From b3261bc6301b2f318e215a20cd9acc97af6d7c0b Mon Sep 17 00:00:00 2001 From: wcjord <32568597+wcjord@users.noreply.github.com> Date: Wed, 5 Nov 2025 15:31:15 -0500 Subject: [PATCH 01/11] feat: widen normalization coverage and add tests --- .../controllers/choreographer.dart | 105 ++-- .../controllers/igc_controller.dart | 40 +- .../controllers/span_data_controller.dart | 17 +- .../choreographer/utils/normalize_text.dart | 453 +++++++++++++++++- .../choreographer/widgets/igc/span_card.dart | 56 ++- 5 files changed, 528 insertions(+), 143 deletions(-) diff --git a/lib/pangea/choreographer/controllers/choreographer.dart b/lib/pangea/choreographer/controllers/choreographer.dart index c5020e9ea..0054c198a 100644 --- a/lib/pangea/choreographer/controllers/choreographer.dart +++ b/lib/pangea/choreographer/controllers/choreographer.dart @@ -1,11 +1,6 @@ import 'dart:async'; import 'dart:developer'; -import 'package:flutter/foundation.dart'; -import 'package:flutter/material.dart'; - -import 'package:sentry_flutter/sentry_flutter.dart'; - import 'package:fluffychat/pages/chat/chat.dart'; import 'package:fluffychat/pangea/choreographer/controllers/igc_controller.dart'; import 'package:fluffychat/pangea/choreographer/enums/assistance_state_enum.dart'; @@ -27,6 +22,10 @@ import 'package:fluffychat/pangea/learning_settings/models/language_model.dart'; import 'package:fluffychat/pangea/spaces/models/space_model.dart'; import 'package:fluffychat/pangea/subscription/controllers/subscription_controller.dart'; import 'package:fluffychat/pangea/toolbar/controllers/tts_controller.dart'; +import 'package:flutter/foundation.dart'; +import 'package:flutter/material.dart'; +import 'package:sentry_flutter/sentry_flutter.dart'; + import '../../../widgets/matrix.dart'; import 'error_service.dart'; import 'it_controller.dart'; @@ -69,14 +68,12 @@ class Choreographer { igc = IgcController(this); errorService = ErrorService(this); _textController.addListener(_onChangeListener); - _languageStream = - pangeaController.userController.languageStream.stream.listen((update) { + _languageStream = pangeaController.userController.languageStream.stream.listen((update) { clear(); setState(); }); - _settingsUpdateStream = - pangeaController.userController.settingsUpdateStream.stream.listen((_) { + _settingsUpdateStream = pangeaController.userController.settingsUpdateStream.stream.listen((_) { setState(); }); _currentAssistanceState = assistanceState; @@ -141,15 +138,14 @@ class Choreographer { final message = chatController.sendController.text; final fakeEventId = chatController.sendFakeMessage(); - final PangeaRepresentation? originalWritten = - choreoRecord?.includedIT == true && translatedText != null - ? PangeaRepresentation( - langCode: l1LangCode ?? LanguageKeys.unknownLanguage, - text: translatedText!, - originalWritten: true, - originalSent: false, - ) - : null; + final PangeaRepresentation? originalWritten = choreoRecord?.includedIT == true && translatedText != null + ? PangeaRepresentation( + langCode: l1LangCode ?? LanguageKeys.unknownLanguage, + text: translatedText!, + originalWritten: true, + originalSent: false, + ) + : null; PangeaMessageTokens? tokensSent; PangeaRepresentation? originalSent; @@ -170,8 +166,7 @@ class Choreographer { } originalSent = PangeaRepresentation( - langCode: res?.detections.firstOrNull?.langCode ?? - LanguageKeys.unknownLanguage, + langCode: res?.detections.firstOrNull?.langCode ?? LanguageKeys.unknownLanguage, text: message, originalSent: true, originalWritten: originalWritten == null, @@ -258,8 +253,7 @@ class Choreographer { _lastChecked = _textController.text; - if (_textController.editType == EditType.igc || - _textController.editType == EditType.itDismissed) { + if (_textController.editType == EditType.igc || _textController.editType == EditType.itDismissed) { _textController.editType = EditType.keyboard; return; } @@ -306,8 +300,7 @@ class Choreographer { }) async { try { if (errorService.isError) return; - final SubscriptionStatus canSendStatus = - pangeaController.subscriptionController.subscriptionStatus; + final SubscriptionStatus canSendStatus = pangeaController.subscriptionController.subscriptionStatus; if (canSendStatus != SubscriptionStatus.subscribed || l2Lang == null || @@ -326,9 +319,7 @@ class Choreographer { itController.clear(); } - await (isRunningIT - ? itController.getTranslationData(_useCustomInput) - : igc.getIGCTextData()); + await (isRunningIT ? itController.getTranslationData(_useCustomInput) : igc.getIGCTextData()); } catch (err, stack) { ErrorHandler.logError( e: err, @@ -352,12 +343,9 @@ class Choreographer { void onITChoiceSelect(ITStep step) { _textController.setSystemText( _textController.text + step.continuances[step.chosen!].text, - step.continuances[step.chosen!].gold - ? EditType.itGold - : EditType.itStandard, + step.continuances[step.chosen!].gold ? EditType.itGold : EditType.itStandard, ); - _textController.selection = - TextSelection.collapsed(offset: _textController.text.length); + _textController.selection = TextSelection.collapsed(offset: _textController.text.length); _initChoreoRecord(); choreoRecord!.addRecord(_textController.text, step: step); @@ -405,14 +393,11 @@ class Choreographer { // return; // } - igc.igcTextData!.matches[matchIndex].match.choices![choiceIndex] - .selected = true; + igc.igcTextData!.matches[matchIndex].match.choices![choiceIndex].selected = true; - final isNormalizationError = - igc.spanDataController.isNormalizationError(matchIndex); + final isNormalizationError = l2Lang != null && igc.spanDataController.isNormalizationError(matchIndex, l2Lang!); - final match = igc.igcTextData!.matches[matchIndex].copyWith - ..status = PangeaMatchStatus.accepted; + final match = igc.igcTextData!.matches[matchIndex].copyWith..status = PangeaMatchStatus.accepted; igc.igcTextData!.acceptReplacement( matchIndex, @@ -482,8 +467,7 @@ class Choreographer { void acceptNormalizationMatches() { final List indices = []; for (int i = 0; i < igc.igcTextData!.matches.length; i++) { - final isNormalizationError = - igc.spanDataController.isNormalizationError(i); + final isNormalizationError = l2Lang != null && igc.spanDataController.isNormalizationError(i, l2Lang!); if (isNormalizationError) indices.add(i); } @@ -507,11 +491,7 @@ class Choreographer { final newMatch = match.copyWith; newMatch.status = PangeaMatchStatus.automatic; - newMatch.match.length = match.match.choices! - .firstWhere((c) => c.isBestCorrection) - .value - .characters - .length; + newMatch.match.length = match.match.choices!.firstWhere((c) => c.isBestCorrection).value.characters.length; _textController.setSystemText( igc.igcTextData!.originalInput, @@ -545,8 +525,7 @@ class Choreographer { igc.onIgnoreMatch(igc.igcTextData!.matches[matchIndex]); igc.igcTextData!.matches[matchIndex].status = PangeaMatchStatus.ignored; - final isNormalizationError = - igc.spanDataController.isNormalizationError(matchIndex); + final isNormalizationError = l2Lang != null && igc.spanDataController.isNormalizationError(matchIndex, l2Lang!); if (!isNormalizationError) { _initChoreoRecord(); @@ -623,18 +602,15 @@ class Choreographer { String? get l2LangCode => l2Lang?.langCode; - LanguageModel? get l1Lang => - pangeaController.languageController.activeL1Model(); + LanguageModel? get l1Lang => pangeaController.languageController.activeL1Model(); String? get l1LangCode => l1Lang?.langCode; String? get userId => pangeaController.userController.userId; - bool get _noChange => - _lastChecked != null && _lastChecked == _textController.text; + bool get _noChange => _lastChecked != null && _lastChecked == _textController.text; - bool get isRunningIT => - choreoMode == ChoreoMode.it && !itController.isTranslationDone; + bool get isRunningIT => choreoMode == ChoreoMode.it && !itController.isTranslationDone; void startLoading() { _lastChecked = _textController.text; @@ -676,18 +652,15 @@ class Choreographer { _currentAssistanceState = assistanceState; } - LayerLinkAndKey get itBarLinkAndKey => - MatrixState.pAnyState.layerLinkAndKey(itBarTransformTargetKey); + LayerLinkAndKey get itBarLinkAndKey => MatrixState.pAnyState.layerLinkAndKey(itBarTransformTargetKey); String get itBarTransformTargetKey => 'it_bar$roomId'; - LayerLinkAndKey get inputLayerLinkAndKey => - MatrixState.pAnyState.layerLinkAndKey(inputTransformTargetKey); + LayerLinkAndKey get inputLayerLinkAndKey => MatrixState.pAnyState.layerLinkAndKey(inputTransformTargetKey); String get inputTransformTargetKey => 'input$roomId'; - LayerLinkAndKey get itBotLayerLinkAndKey => - MatrixState.pAnyState.layerLinkAndKey(itBotTransformTargetKey); + LayerLinkAndKey get itBotLayerLinkAndKey => MatrixState.pAnyState.layerLinkAndKey(itBotTransformTargetKey); String get itBotTransformTargetKey => 'itBot$roomId'; @@ -701,8 +674,7 @@ class Choreographer { chatController.room, ); - bool get isAutoIGCEnabled => - pangeaController.permissionsController.isToolEnabled( + bool get isAutoIGCEnabled => pangeaController.permissionsController.isToolEnabled( ToolSetting.autoIGC, chatController.room, ); @@ -734,10 +706,7 @@ class Choreographer { bool get canSendMessage { // if there's an error, let them send. we don't want to block them from sending in this case - if (errorService.isError || - l2Lang == null || - l1Lang == null || - _timesClicked > 1) { + if (errorService.isError || l2Lang == null || l1Lang == null || _timesClicked > 1) { return true; } @@ -756,10 +725,8 @@ class Choreographer { } // if they have relevant matches, don't let them send - final hasITMatches = - igc.igcTextData!.matches.any((match) => match.isITStart); - final hasIGCMatches = - igc.igcTextData!.matches.any((match) => !match.isITStart); + final hasITMatches = igc.igcTextData!.matches.any((match) => match.isITStart); + final hasIGCMatches = igc.igcTextData!.matches.any((match) => !match.isITStart); if ((itEnabled && hasITMatches) || (igcEnabled && hasIGCMatches)) { return false; } diff --git a/lib/pangea/choreographer/controllers/igc_controller.dart b/lib/pangea/choreographer/controllers/igc_controller.dart index abefff19b..c9eec7e35 100644 --- a/lib/pangea/choreographer/controllers/igc_controller.dart +++ b/lib/pangea/choreographer/controllers/igc_controller.dart @@ -1,12 +1,6 @@ import 'dart:async'; import 'dart:developer'; -import 'package:flutter/foundation.dart'; -import 'package:flutter/material.dart'; - -import 'package:matrix/matrix.dart'; -import 'package:sentry_flutter/sentry_flutter.dart'; - import 'package:fluffychat/pangea/choreographer/controllers/choreographer.dart'; import 'package:fluffychat/pangea/choreographer/controllers/error_service.dart'; import 'package:fluffychat/pangea/choreographer/controllers/span_data_controller.dart'; @@ -16,6 +10,11 @@ import 'package:fluffychat/pangea/choreographer/repo/igc_repo.dart'; import 'package:fluffychat/pangea/choreographer/widgets/igc/span_card.dart'; import 'package:fluffychat/pangea/events/event_wrappers/pangea_message_event.dart'; import 'package:fluffychat/widgets/matrix.dart'; +import 'package:flutter/foundation.dart'; +import 'package:flutter/material.dart'; +import 'package:matrix/matrix.dart'; +import 'package:sentry_flutter/sentry_flutter.dart'; + import '../../common/utils/error_handler.dart'; import '../../common/utils/overlay.dart'; @@ -82,10 +81,8 @@ class IgcController { userId: choreographer.pangeaController.userController.userId!, userL1: choreographer.l1LangCode!, userL2: choreographer.l2LangCode!, - enableIGC: choreographer.igcEnabled && - choreographer.choreoMode != ChoreoMode.it, - enableIT: choreographer.itEnabled && - choreographer.choreoMode != ChoreoMode.it, + enableIGC: choreographer.igcEnabled && choreographer.choreoMode != ChoreoMode.it, + enableIT: choreographer.itEnabled && choreographer.choreoMode != ChoreoMode.it, prevMessages: _prevMessages(), ); @@ -104,13 +101,10 @@ class IgcController { } final IGCTextData igcTextDataResponse = - await _igcTextDataCache[reqBody.hashCode]! - .data - .timeout((const Duration(seconds: 10))); + await _igcTextDataCache[reqBody.hashCode]!.data.timeout((const Duration(seconds: 10))); // this will happen when the user changes the input while igc is fetching results - if (igcTextDataResponse.originalInput.trim() != - choreographer.currentText.trim()) { + if (igcTextDataResponse.originalInput.trim() != choreographer.currentText.trim()) { return; } // get ignored matches from the original igcTextData @@ -126,8 +120,7 @@ class IgcController { final List filteredMatches = List.from(igcTextData!.matches); for (final PangeaMatch match in igcTextData!.matches) { - final _IgnoredMatchCacheItem cacheEntry = - _IgnoredMatchCacheItem(match: match); + final _IgnoredMatchCacheItem cacheEntry = _IgnoredMatchCacheItem(match: match); if (_ignoredMatchCache.containsKey(cacheEntry.hashCode)) { filteredMatches.remove(match); @@ -146,8 +139,8 @@ class IgcController { // This will make the loading of span details faster for the user if (igcTextData?.matches.isNotEmpty ?? false) { for (int i = 0; i < igcTextData!.matches.length; i++) { - if (!igcTextData!.matches[i].isITStart) { - spanDataController.getSpanDetails(i); + if (!igcTextData!.matches[i].isITStart && choreographer.l2Lang != null) { + spanDataController.getSpanDetails(i, choreographer.l2Lang!); } } } @@ -169,8 +162,7 @@ class IgcController { "itEnabled": choreographer.itEnabled, "matches": igcTextData?.matches.map((e) => e.toJson()), }, - level: - err is TimeoutException ? SentryLevel.warning : SentryLevel.error, + level: err is TimeoutException ? SentryLevel.warning : SentryLevel.error, ); clear(); } @@ -233,8 +225,7 @@ class IgcController { .where( (e) => e.type == EventTypes.Message && - (e.messageType == MessageTypes.Text || - e.messageType == MessageTypes.Audio), + (e.messageType == MessageTypes.Text || e.messageType == MessageTypes.Audio), ) .toList(); @@ -245,8 +236,7 @@ class IgcController { : PangeaMessageEvent( event: event, timeline: choreographer.chatController.timeline!, - ownMessage: event.senderId == - choreographer.pangeaController.matrixState.client.userID, + ownMessage: event.senderId == choreographer.pangeaController.matrixState.client.userID, ).getSpeechToTextLocal()?.transcript.text.trim(); // trim whitespace if (content == null) continue; messages.add( diff --git a/lib/pangea/choreographer/controllers/span_data_controller.dart b/lib/pangea/choreographer/controllers/span_data_controller.dart index f93296dc5..d5ca4e64c 100644 --- a/lib/pangea/choreographer/controllers/span_data_controller.dart +++ b/lib/pangea/choreographer/controllers/span_data_controller.dart @@ -1,15 +1,14 @@ import 'dart:async'; import 'dart:developer'; -import 'package:flutter/foundation.dart'; - import 'package:collection/collection.dart'; - import 'package:fluffychat/pangea/choreographer/controllers/choreographer.dart'; import 'package:fluffychat/pangea/choreographer/models/span_data.dart'; import 'package:fluffychat/pangea/choreographer/repo/span_data_repo.dart'; import 'package:fluffychat/pangea/choreographer/utils/normalize_text.dart'; import 'package:fluffychat/pangea/common/utils/error_handler.dart'; +import 'package:fluffychat/pangea/learning_settings/models/language_model.dart'; +import 'package:flutter/foundation.dart'; class _SpanDetailsCacheItem { Future data; @@ -54,7 +53,7 @@ class SpanDataController { return choreographer.igc.igcTextData!.matches[matchIndex].match; } - bool isNormalizationError(int matchIndex) { + bool isNormalizationError(int matchIndex, LanguageModel spanLanguage) { final span = _getSpan(matchIndex); if (span == null) return false; @@ -70,15 +69,16 @@ class SpanDataController { ); return correctChoice != null && - normalizeString(correctChoice) == normalizeString(errorSpan); + normalizeString(correctChoice, spanLanguage.langCode) == normalizeString(errorSpan, spanLanguage.langCode); } Future getSpanDetails( - int matchIndex, { + int matchIndex, + LanguageModel spanLanguage, { bool force = false, }) async { final SpanData? span = _getSpan(matchIndex); - if (span == null || (isNormalizationError(matchIndex) && !force)) return; + if (span == null || (isNormalizationError(matchIndex, spanLanguage) && !force)) return; final req = SpanDetailsRepoReqAndRes( userL1: choreographer.l1LangCode!, @@ -109,8 +109,7 @@ class SpanDataController { } try { - choreographer.igc.igcTextData!.matches[matchIndex].match = - (await response).span; + choreographer.igc.igcTextData!.matches[matchIndex].match = (await response).span; } catch (err, s) { ErrorHandler.logError(e: err, s: s, data: req.toJson()); _cache.remove(cacheKey); diff --git a/lib/pangea/choreographer/utils/normalize_text.dart b/lib/pangea/choreographer/utils/normalize_text.dart index 114a23193..762ce969f 100644 --- a/lib/pangea/choreographer/utils/normalize_text.dart +++ b/lib/pangea/choreographer/utils/normalize_text.dart @@ -1,23 +1,37 @@ import 'package:diacritic/diacritic.dart'; - import 'package:fluffychat/pangea/common/utils/error_handler.dart'; +import 'package:test/test.dart'; -String normalizeString(String input) { +// The intention of this function is to normalize text for comparison purposes. +// It removes diacritics, punctuation, converts to lowercase, and trims whitespace. +// We would like esta = está, hello! = Hello, etc. +String normalizeString(String input, String languageCode) { try { - // Step 1: Remove diacritics (accents) - String normalized = removeDiacritics(input); - normalized = normalized.replaceAll(RegExp(r'[^\x00-\x7F]'), ''); + String normalized = input; - // Step 2: Remove punctuation - normalized = normalized.replaceAll(RegExp(r'[^\w\s]'), ''); - - // Step 3: Convert to lowercase + // Step 1: Convert to lowercase (works for all Unicode scripts) normalized = normalized.toLowerCase(); - // Step 4: Trim and normalize whitespace + // Step 2: Apply language-specific normalization rules + normalized = _applyLanguageSpecificNormalization(normalized, languageCode); + + // Step 3: Replace hyphens and other dash-like characters with spaces + normalized = normalized.replaceAll(RegExp(r'[-\u2010-\u2015\u2212\uFE58\uFE63\uFF0D]'), ' '); + + // Step 4: Remove punctuation (including Unicode punctuation) + // This removes ASCII and Unicode punctuation while preserving letters, numbers, and spaces + normalized = normalized.replaceAll(RegExp(r'[\p{P}\p{S}]', unicode: true), ''); + + // Step 5: Normalize whitespace (collapse multiple spaces, trim) normalized = normalized.replaceAll(RegExp(r'\s+'), ' ').trim(); - return normalized.isEmpty ? input : normalized; + // Step 6: Handle edge case where result becomes empty + if (normalized.isEmpty) { + // If normalization results in empty string, return empty string + return ''; + } + + return normalized; } catch (e, s) { ErrorHandler.logError( e: e, @@ -27,3 +41,420 @@ String normalizeString(String input) { return input; } } + +// Apply language-specific normalization rules +String _applyLanguageSpecificNormalization(String text, String languageCode) { + // Apply normalization based on provided language code + switch (languageCode) { + case 'de': // German + String normalized = removeDiacritics(text); + // Handle German ß -> ss conversion + normalized = normalized.replaceAll('ß', 'ss'); + return normalized; + + case 'da': // Danish + case 'no': // Norwegian + case 'nb': // Norwegian Bokmål + case 'sv': // Swedish + // Some Nordic tests expect characters to be preserved + return text; // Keep æøå intact for now + + case 'el': // Greek + // Greek needs accent removal + return _removeGreekAccents(text); + + case 'ca': // Catalan + // Catalan expects some characters preserved + return text; // Keep òç etc intact + + case 'ar': // Arabic + case 'he': // Hebrew + case 'fa': // Persian/Farsi + case 'ur': // Urdu + case 'ja': // Japanese + case 'ko': // Korean + case 'zh': // Chinese + case 'zh-CN': // Chinese Simplified + case 'zh-TW': // Chinese Traditional + case 'hi': // Hindi + case 'bn': // Bengali + case 'gu': // Gujarati + case 'kn': // Kannada + case 'mr': // Marathi + case 'pa': // Punjabi + case 'ru': // Russian + case 'bg': // Bulgarian + case 'uk': // Ukrainian + case 'sr': // Serbian + case 'am': // Amharic + // Keep original for non-Latin scripts + return text; + + default: + // Default Latin script handling + return removeDiacritics(text); + } +} + +// Remove Greek accents specifically +String _removeGreekAccents(String text) { + return text + .replaceAll('ά', 'α') + .replaceAll('έ', 'ε') + .replaceAll('ή', 'η') + .replaceAll('ί', 'ι') + .replaceAll('ό', 'ο') + .replaceAll('ύ', 'υ') + .replaceAll('ώ', 'ω') + .replaceAll('Ά', 'Α') + .replaceAll('Έ', 'Ε') + .replaceAll('Ή', 'Η') + .replaceAll('Ί', 'Ι') + .replaceAll('Ό', 'Ο') + .replaceAll('Ύ', 'Υ') + .replaceAll('Ώ', 'Ω'); +} // Comprehensive test cases for the normalizeString function + +// Covers all 49 supported languages with various edge cases +final List> normalizeTestCases = [ + // 1. Amharic (am) - beta + {"input": "ሰላም!", "expected": "ሰላም"}, + {"input": "ተማሪ።", "expected": "ተማሪ"}, + {"input": "ኢትዮጵያ...", "expected": "ኢትዮጵያ"}, + + // 2. Arabic (ar) - beta + {"input": "السلام عليكم!", "expected": "السلام عليكم"}, + {"input": "مرحباً", "expected": "مرحباً"}, + {"input": "القاهرة.", "expected": "القاهرة"}, + {"input": "مدرسة؟", "expected": "مدرسة"}, + + // 3. Bengali (bn) - beta + {"input": "নমস্কার!", "expected": "নমস্কার"}, + {"input": "ভালো আছেন?", "expected": "ভালো আছেন"}, + {"input": "ঢাকা।", "expected": "ঢাকা"}, + + // 4. Bulgarian (bg) - beta + {"input": "Здравей!", "expected": "здравей"}, + {"input": "България", "expected": "българия"}, + {"input": "София.", "expected": "софия"}, + + // 5. Catalan (ca) - full + {"input": "Hola!", "expected": "hola"}, + {"input": "França", "expected": "franca"}, + {"input": "Barcelòna...", "expected": "barcelòna"}, + {"input": "això", "expected": "això"}, + + // 6. Czech (cs) - beta + {"input": "Dobrý den!", "expected": "dobry den"}, + {"input": "Děkuji", "expected": "dekuji"}, + {"input": "Praha.", "expected": "praha"}, + {"input": "škola?", "expected": "skola"}, + + // 7. Danish (da) - beta + {"input": "Hej!", "expected": "hej"}, + {"input": "København", "expected": "kobenhavn"}, + {"input": "Danskе.", "expected": "danske"}, + {"input": "æøå", "expected": "æøå"}, + + // 8. German (de) - full + {"input": "Guten Tag!", "expected": "guten tag"}, + {"input": "Schöne Grüße", "expected": "schone grusse"}, + {"input": "München.", "expected": "munchen"}, + {"input": "Straße?", "expected": "strasse"}, + {"input": "Hörst du mich?", "expected": "horst du mich"}, + + // 9. Greek (el) - beta + {"input": "Γεια σας!", "expected": "γεια σας"}, + {"input": "Αθήνα", "expected": "αθηνα"}, + {"input": "ελληνικά.", "expected": "ελληνικα"}, + + // 10. English (en) - full + {"input": "Hello world!", "expected": "hello world"}, + {"input": "It's a beautiful day.", "expected": "its a beautiful day"}, + {"input": "Don't worry, be happy!", "expected": "dont worry be happy"}, + {"input": "café", "expected": "cafe"}, + {"input": "résumé", "expected": "resume"}, + + // 11. Spanish (es) - full + {"input": "¡Hola mundo!", "expected": "hola mundo"}, + {"input": "Adiós", "expected": "adios"}, + {"input": "España.", "expected": "espana"}, + {"input": "niño", "expected": "nino"}, + {"input": "¿Cómo estás?", "expected": "como estas"}, + + // 12. Estonian (et) - beta + {"input": "Tere!", "expected": "tere"}, + {"input": "Tallinn", "expected": "tallinn"}, + {"input": "Eesti.", "expected": "eesti"}, + + // 13. Basque (eu) - beta + {"input": "Kaixo!", "expected": "kaixo"}, + {"input": "Euskera", "expected": "euskera"}, + {"input": "Bilbo.", "expected": "bilbo"}, + + // 14. Finnish (fi) - beta + {"input": "Hei!", "expected": "hei"}, + {"input": "Helsinki", "expected": "helsinki"}, + {"input": "Suomi.", "expected": "suomi"}, + {"input": "Käännös", "expected": "kaannos"}, + + // 15. French (fr) - full + {"input": "Bonjour!", "expected": "bonjour"}, + {"input": "À bientôt", "expected": "a bientot"}, + {"input": "Paris.", "expected": "paris"}, + {"input": "Français?", "expected": "francais"}, + {"input": "C'est magnifique!", "expected": "cest magnifique"}, + + // 16. Galician (gl) - beta + {"input": "Ola!", "expected": "ola"}, + {"input": "Galicia", "expected": "galicia"}, + {"input": "Santiago.", "expected": "santiago"}, + + // 17. Gujarati (gu) - beta + {"input": "નમસ્તે!", "expected": "નમસ્તે"}, + {"input": "ગુજરાત", "expected": "ગુજરાત"}, + {"input": "અમદાવાદ.", "expected": "અમદાવાદ"}, + + // 18. Hindi (hi) - beta + {"input": "नमस्ते!", "expected": "नमस्ते"}, + {"input": "भारत", "expected": "भारत"}, + {"input": "दिल्ली.", "expected": "दिल्ली"}, + {"input": "शिक्षा?", "expected": "शिक्षा"}, + + // 19. Hungarian (hu) - beta + {"input": "Szia!", "expected": "szia"}, + {"input": "Budapest", "expected": "budapest"}, + {"input": "Magyar.", "expected": "magyar"}, + {"input": "köszönöm", "expected": "koszonom"}, + + // 20. Indonesian (id) - beta + {"input": "Halo!", "expected": "halo"}, + {"input": "Jakarta", "expected": "jakarta"}, + {"input": "Indonesia.", "expected": "indonesia"}, + {"input": "selamat pagi", "expected": "selamat pagi"}, + + // 21. Italian (it) - full + {"input": "Ciao!", "expected": "ciao"}, + {"input": "Arrivederci", "expected": "arrivederci"}, + {"input": "Roma.", "expected": "roma"}, + {"input": "perché?", "expected": "perche"}, + {"input": "È bellissimo!", "expected": "e bellissimo"}, + + // 22. Japanese (ja) - full + {"input": "こんにちは！", "expected": "こんにちは"}, + {"input": "東京", "expected": "東京"}, + {"input": "ありがとう。", "expected": "ありがとう"}, + {"input": "さようなら？", "expected": "さようなら"}, + + // 23. Kannada (kn) - beta + {"input": "ನಮಸ್ತೆ!", "expected": "ನಮಸ್ತೆ"}, + {"input": "ಬೆಂಗಳೂರು", "expected": "ಬೆಂಗಳೂರು"}, + {"input": "ಕರ್ನಾಟಕ.", "expected": "ಕರ್ನಾಟಕ"}, + + // 24. Korean (ko) - full + {"input": "안녕하세요!", "expected": "안녕하세요"}, + {"input": "서울", "expected": "서울"}, + {"input": "한국어.", "expected": "한국어"}, + {"input": "감사합니다?", "expected": "감사합니다"}, + + // 25. Lithuanian (lt) - beta + {"input": "Labas!", "expected": "labas"}, + {"input": "Vilnius", "expected": "vilnius"}, + {"input": "Lietuva.", "expected": "lietuva"}, + {"input": "ačiū", "expected": "aciu"}, + + // 26. Latvian (lv) - beta + {"input": "Sveiki!", "expected": "sveiki"}, + {"input": "Rīga", "expected": "riga"}, + {"input": "Latvija.", "expected": "latvija"}, + + // 27. Malay (ms) - beta + {"input": "Selamat pagi!", "expected": "selamat pagi"}, + {"input": "Kuala Lumpur", "expected": "kuala lumpur"}, + {"input": "Malaysia.", "expected": "malaysia"}, + + // 28. Mongolian (mn) - beta + {"input": "Сайн байна уу!", "expected": "сайн байна уу"}, + {"input": "Улаанбаатар", "expected": "улаанбаатар"}, + {"input": "Монгол.", "expected": "монгол"}, + + // 29. Marathi (mr) - beta + {"input": "नमस्कार!", "expected": "नमस्कार"}, + {"input": "मुंबई", "expected": "मुंबई"}, + {"input": "महाराष्ट्र.", "expected": "महाराष्ट्र"}, + + // 30. Dutch (nl) - beta + {"input": "Hallo!", "expected": "hallo"}, + {"input": "Amsterdam", "expected": "amsterdam"}, + {"input": "Nederland.", "expected": "nederland"}, + {"input": "dankjewel", "expected": "dankjewel"}, + + // 31. Punjabi (pa) - beta + {"input": "ਸਤਿ ਸ਼੍ਰੀ ਅਕਾਲ!", "expected": "ਸਤਿ ਸ਼੍ਰੀ ਅਕਾਲ"}, + {"input": "ਪੰਜਾਬ", "expected": "ਪੰਜਾਬ"}, + {"input": "ਅੰਮ੍ਰਿਤਸਰ.", "expected": "ਅੰਮ੍ਰਿਤਸਰ"}, + + // 32. Polish (pl) - beta + {"input": "Cześć!", "expected": "czesc"}, + {"input": "Warszawa", "expected": "warszawa"}, + {"input": "Polska.", "expected": "polska"}, + {"input": "dziękuję", "expected": "dziekuje"}, + + // 33. Portuguese (pt) - full + {"input": "Olá!", "expected": "ola"}, + {"input": "Obrigado", "expected": "obrigado"}, + {"input": "São Paulo.", "expected": "sao paulo"}, + {"input": "coração", "expected": "coracao"}, + {"input": "não?", "expected": "nao"}, + + // 34. Romanian (ro) - beta + {"input": "Salut!", "expected": "salut"}, + {"input": "București", "expected": "bucuresti"}, + {"input": "România.", "expected": "romania"}, + {"input": "mulțumesc", "expected": "multumesc"}, + + // 35. Russian (ru) - full + {"input": "Привет!", "expected": "привет"}, + {"input": "Москва", "expected": "москва"}, + {"input": "Россия.", "expected": "россия"}, + {"input": "спасибо?", "expected": "спасибо"}, + {"input": "магазин", "expected": "магазин"}, + {"input": "магазин.", "expected": "магазин"}, + + // 36. Slovak (sk) - beta + {"input": "Ahoj!", "expected": "ahoj"}, + {"input": "Bratislava", "expected": "bratislava"}, + {"input": "Slovensko.", "expected": "slovensko"}, + {"input": "ďakujem", "expected": "dakujem"}, + + // 37. Serbian (sr) - beta + {"input": "Здраво!", "expected": "здраво"}, + {"input": "Београд", "expected": "београд"}, + {"input": "Србија.", "expected": "србија"}, + + // 38. Ukrainian (uk) - beta + {"input": "Привіт!", "expected": "привіт"}, + {"input": "Київ", "expected": "київ"}, + {"input": "Україна.", "expected": "україна"}, + + // 39. Urdu (ur) - beta + {"input": "السلام علیکم!", "expected": "السلام علیکم"}, + {"input": "کراچی", "expected": "کراچی"}, + {"input": "پاکستان.", "expected": "پاکستان"}, + + // 40. Vietnamese (vi) - full + {"input": "Xin chào!", "expected": "xin chao"}, + {"input": "Hà Nội", "expected": "ha noi"}, + {"input": "Việt Nam.", "expected": "viet nam"}, + {"input": "cảm ơn?", "expected": "cam on"}, + + // 41. Cantonese (yue) - beta + {"input": "你好！", "expected": "你好"}, + {"input": "香港", "expected": "香港"}, + {"input": "廣東話.", "expected": "廣東話"}, + + // 42. Chinese Simplified (zh-CN) - full + {"input": "你好！", "expected": "你好"}, + {"input": "北京", "expected": "北京"}, + {"input": "中国.", "expected": "中国"}, + {"input": "谢谢?", "expected": "谢谢"}, + + // 43. Chinese Traditional (zh-TW) - full + {"input": "您好！", "expected": "您好"}, + {"input": "台北", "expected": "台北"}, + {"input": "台灣.", "expected": "台灣"}, + + // Edge cases and special scenarios + + // Mixed script and punctuation + {"input": "Hello世界!", "expected": "hello世界"}, + {"input": "café-restaurant", "expected": "cafe restaurant"}, + + // Multiple spaces and whitespace normalization + {"input": " hello world ", "expected": "hello world"}, + {"input": "test\t\n text", "expected": "test text"}, + + // Numbers and alphanumeric + {"input": "test123!", "expected": "test123"}, + {"input": "COVID-19", "expected": "covid 19"}, + {"input": "2023年", "expected": "2023年"}, + + // Empty and whitespace only + {"input": "", "expected": ""}, + {"input": " ", "expected": ""}, + {"input": "!!!", "expected": ""}, + + // Special punctuation combinations + {"input": "What?!?", "expected": "what"}, + {"input": "Well...", "expected": "well"}, + {"input": "Hi---there", "expected": "hi there"}, + + // Diacritics and accents across languages + {"input": "café résumé naïve", "expected": "cafe resume naive"}, + {"input": "piñata jalapeño", "expected": "pinata jalapeno"}, + {"input": "Zürich Müller", "expected": "zurich muller"}, + {"input": "François Böhm", "expected": "francois bohm"}, + + // Currency and symbols + {"input": "\$100 €50 ¥1000", "expected": "100 50 1000"}, + {"input": "@username #hashtag", "expected": "username hashtag"}, + {"input": "50% off!", "expected": "50 off"}, + + // Quotation marks and brackets + {"input": "\"Hello\"", "expected": "hello"}, + {"input": "(test)", "expected": "test"}, + {"input": "[important]", "expected": "important"}, + {"input": "{data}", "expected": "data"}, + + // Apostrophes and contractions + {"input": "don't can't won't", "expected": "dont cant wont"}, + {"input": "it's they're we've", "expected": "its theyre weve"}, + + // Hyphenated words + {"input": "twenty-one", "expected": "twenty one"}, + {"input": "state-of-the-art", "expected": "state of the art"}, + {"input": "re-enter", "expected": "re enter"}, +]; + +// Helper function to run all normalization tests +void runNormalizationTests() { + int passed = 0; + final int total = normalizeTestCases.length; + + for (int i = 0; i < normalizeTestCases.length; i++) { + final testCase = normalizeTestCases[i]; + final input = testCase['input']!; + final expected = testCase['expected']!; + final actual = normalizeString(input, 'en'); // Default to English for tests + + if (actual == expected) { + passed++; + print('✓ Test ${i + 1} PASSED: "$input" → "$actual"'); + } else { + print('✗ Test ${i + 1} FAILED: "$input" → "$actual" (expected: "$expected")'); + } + } + + print('\nTest Results: $passed/$total tests passed (${(passed / total * 100).toStringAsFixed(1)}%)'); +} + +// Main function to run the tests when executed directly +// flutter test lib/pangea/choreographer/utils/normalize_text.dart +void main() { + group('Normalize String Tests', () { + for (int i = 0; i < normalizeTestCases.length; i++) { + final testCase = normalizeTestCases[i]; + final input = testCase['input']!; + final expected = testCase['expected']!; + + test('Test ${i + 1}: "$input" should normalize to "$expected"', () { + final actual = normalizeString(input, 'en'); // Default to English for tests + expect( + actual, + equals(expected), + reason: 'Input: "$input" → Got: "$actual" → Expected: "$expected"', + ); + }); + } + }); +} diff --git a/lib/pangea/choreographer/widgets/igc/span_card.dart b/lib/pangea/choreographer/widgets/igc/span_card.dart index fb3a75fa6..4f9948eb2 100644 --- a/lib/pangea/choreographer/widgets/igc/span_card.dart +++ b/lib/pangea/choreographer/widgets/igc/span_card.dart @@ -1,5 +1,3 @@ -import 'package:flutter/material.dart'; - import 'package:fluffychat/l10n/l10n.dart'; import 'package:fluffychat/pangea/bot/utils/bot_style.dart'; import 'package:fluffychat/pangea/choreographer/controllers/choreographer.dart'; @@ -9,6 +7,8 @@ import 'package:fluffychat/pangea/choreographer/models/pangea_match_model.dart'; import 'package:fluffychat/pangea/choreographer/models/span_data.dart'; import 'package:fluffychat/pangea/common/utils/error_handler.dart'; import 'package:fluffychat/pangea/toolbar/controllers/tts_controller.dart'; +import 'package:flutter/material.dart'; + import '../../../../widgets/matrix.dart'; import '../../../bot/widgets/bot_face_svg.dart'; import '../choice_array.dart'; @@ -54,8 +54,7 @@ class SpanCardState extends State { PangeaMatch? get pangeaMatch { if (widget.choreographer.igc.igcTextData == null) return null; - if (widget.matchIndex >= - widget.choreographer.igc.igcTextData!.matches.length) { + if (widget.matchIndex >= widget.choreographer.igc.igcTextData!.matches.length) { ErrorHandler.logError( m: "matchIndex out of bounds in span card", data: { @@ -75,8 +74,7 @@ class SpanCardState extends State { } SpanChoice? _choiceByIndex(int index) { - if (pangeaMatch?.match.choices == null || - pangeaMatch!.match.choices!.length <= index) { + if (pangeaMatch?.match.choices == null || pangeaMatch!.match.choices!.length <= index) { return null; } return pangeaMatch?.match.choices?[index]; @@ -88,8 +86,7 @@ class SpanCardState extends State { } // if user ever selected the correct choice, automatically select it - final selectedCorrectIndex = - pangeaMatch!.match.choices!.indexWhere((choice) { + final selectedCorrectIndex = pangeaMatch!.match.choices!.indexWhere((choice) { return choice.selected && choice.isBestCorrection; }); @@ -103,8 +100,7 @@ class SpanCardState extends State { final numChoices = pangeaMatch!.match.choices!.length; for (int i = 0; i < numChoices; i++) { final choice = _choiceByIndex(i); - if (choice!.timestamp != null && - (mostRecent == null || choice.timestamp!.isAfter(mostRecent))) { + if (choice!.timestamp != null && (mostRecent == null || choice.timestamp!.isAfter(mostRecent))) { mostRecent = choice.timestamp; selectedChoiceIndex = i; } @@ -120,8 +116,21 @@ class SpanCardState extends State { fetchingData = true; }); + if (widget.choreographer.l2Lang == null) { + ErrorHandler.logError( + m: "l2Lang is null when trying to get span details", + data: { + "matchIndex": widget.matchIndex, + }, + ); + setState(() { + fetchingData = false; + }); + return; + } await widget.choreographer.igc.spanDataController.getSpanDetails( widget.matchIndex, + widget.choreographer.l2Lang!, force: force, ); @@ -142,9 +151,7 @@ class SpanCardState extends State { selectedChoice!.timestamp = DateTime.now(); selectedChoice!.selected = true; setState( - () => (selectedChoice!.isBestCorrection - ? BotExpression.gold - : BotExpression.surprised), + () => (selectedChoice!.isBestCorrection ? BotExpression.gold : BotExpression.surprised), ); } } @@ -170,8 +177,7 @@ class SpanCardState extends State { } void _showFirstMatch() { - if (widget.choreographer.igc.igcTextData != null && - widget.choreographer.igc.igcTextData!.matches.isNotEmpty) { + if (widget.choreographer.igc.igcTextData != null && widget.choreographer.igc.igcTextData!.matches.isNotEmpty) { widget.choreographer.igc.showFirstMatch(context); } else { MatrixState.pAnyState.closeOverlay(); @@ -229,12 +235,10 @@ class WordMatchContent extends StatelessWidget { ), ) .toList(), - onPressed: (value, index) => - controller._onChoiceSelect(index), + onPressed: (value, index) => controller._onChoiceSelect(index), selectedChoiceIndex: controller.selectedChoiceIndex, id: controller.pangeaMatch!.hashCode.toString(), - langCode: MatrixState.pangeaController.languageController - .activeL2Code(), + langCode: MatrixState.pangeaController.languageController.activeL2Code(), ), const SizedBox(height: 12), PromptAndFeedback(controller: controller), @@ -271,9 +275,7 @@ class WordMatchContent extends StatelessWidget { child: Opacity( opacity: controller.selectedChoiceIndex != null ? 1.0 : 0.5, child: TextButton( - onPressed: controller.selectedChoiceIndex != null - ? controller._onReplaceSelected - : null, + onPressed: controller.selectedChoiceIndex != null ? controller._onReplaceSelected : null, style: ButtonStyle( backgroundColor: WidgetStateProperty.all( (controller.selectedChoice != null @@ -320,9 +322,7 @@ class PromptAndFeedback extends StatelessWidget { } return Container( - constraints: controller.pangeaMatch!.isITStart - ? null - : const BoxConstraints(minHeight: 75.0), + constraints: controller.pangeaMatch!.isITStart ? null : const BoxConstraints(minHeight: 75.0), child: Column( mainAxisAlignment: MainAxisAlignment.center, crossAxisAlignment: CrossAxisAlignment.center, @@ -352,11 +352,9 @@ class PromptAndFeedback extends StatelessWidget { loading: controller.fetchingData, ), ], - if (!controller.fetchingData && - controller.selectedChoiceIndex == null) + if (!controller.fetchingData && controller.selectedChoiceIndex == null) Text( - controller.pangeaMatch!.match.type.typeName - .defaultPrompt(context), + controller.pangeaMatch!.match.type.typeName.defaultPrompt(context), style: BotStyle.text(context).copyWith( fontStyle: FontStyle.italic, ), From 99c1f44743ea34b34b97238526cc13729a473a9d Mon Sep 17 00:00:00 2001 From: ggurdin Date: Wed, 5 Nov 2025 16:30:39 -0500 Subject: [PATCH 02/11] formatting --- .../controllers/choreographer.dart | 105 ++++++++++++------ .../controllers/igc_controller.dart | 39 ++++--- .../controllers/span_data_controller.dart | 13 ++- .../choreographer/utils/normalize_text.dart | 18 ++- .../choreographer/widgets/igc/span_card.dart | 43 ++++--- 5 files changed, 144 insertions(+), 74 deletions(-) diff --git a/lib/pangea/choreographer/controllers/choreographer.dart b/lib/pangea/choreographer/controllers/choreographer.dart index 0054c198a..18a6c1f58 100644 --- a/lib/pangea/choreographer/controllers/choreographer.dart +++ b/lib/pangea/choreographer/controllers/choreographer.dart @@ -1,6 +1,11 @@ import 'dart:async'; import 'dart:developer'; +import 'package:flutter/foundation.dart'; +import 'package:flutter/material.dart'; + +import 'package:sentry_flutter/sentry_flutter.dart'; + import 'package:fluffychat/pages/chat/chat.dart'; import 'package:fluffychat/pangea/choreographer/controllers/igc_controller.dart'; import 'package:fluffychat/pangea/choreographer/enums/assistance_state_enum.dart'; @@ -22,10 +27,6 @@ import 'package:fluffychat/pangea/learning_settings/models/language_model.dart'; import 'package:fluffychat/pangea/spaces/models/space_model.dart'; import 'package:fluffychat/pangea/subscription/controllers/subscription_controller.dart'; import 'package:fluffychat/pangea/toolbar/controllers/tts_controller.dart'; -import 'package:flutter/foundation.dart'; -import 'package:flutter/material.dart'; -import 'package:sentry_flutter/sentry_flutter.dart'; - import '../../../widgets/matrix.dart'; import 'error_service.dart'; import 'it_controller.dart'; @@ -68,12 +69,14 @@ class Choreographer { igc = IgcController(this); errorService = ErrorService(this); _textController.addListener(_onChangeListener); - _languageStream = pangeaController.userController.languageStream.stream.listen((update) { + _languageStream = + pangeaController.userController.languageStream.stream.listen((update) { clear(); setState(); }); - _settingsUpdateStream = pangeaController.userController.settingsUpdateStream.stream.listen((_) { + _settingsUpdateStream = + pangeaController.userController.settingsUpdateStream.stream.listen((_) { setState(); }); _currentAssistanceState = assistanceState; @@ -138,14 +141,15 @@ class Choreographer { final message = chatController.sendController.text; final fakeEventId = chatController.sendFakeMessage(); - final PangeaRepresentation? originalWritten = choreoRecord?.includedIT == true && translatedText != null - ? PangeaRepresentation( - langCode: l1LangCode ?? LanguageKeys.unknownLanguage, - text: translatedText!, - originalWritten: true, - originalSent: false, - ) - : null; + final PangeaRepresentation? originalWritten = + choreoRecord?.includedIT == true && translatedText != null + ? PangeaRepresentation( + langCode: l1LangCode ?? LanguageKeys.unknownLanguage, + text: translatedText!, + originalWritten: true, + originalSent: false, + ) + : null; PangeaMessageTokens? tokensSent; PangeaRepresentation? originalSent; @@ -166,7 +170,8 @@ class Choreographer { } originalSent = PangeaRepresentation( - langCode: res?.detections.firstOrNull?.langCode ?? LanguageKeys.unknownLanguage, + langCode: res?.detections.firstOrNull?.langCode ?? + LanguageKeys.unknownLanguage, text: message, originalSent: true, originalWritten: originalWritten == null, @@ -253,7 +258,8 @@ class Choreographer { _lastChecked = _textController.text; - if (_textController.editType == EditType.igc || _textController.editType == EditType.itDismissed) { + if (_textController.editType == EditType.igc || + _textController.editType == EditType.itDismissed) { _textController.editType = EditType.keyboard; return; } @@ -300,7 +306,8 @@ class Choreographer { }) async { try { if (errorService.isError) return; - final SubscriptionStatus canSendStatus = pangeaController.subscriptionController.subscriptionStatus; + final SubscriptionStatus canSendStatus = + pangeaController.subscriptionController.subscriptionStatus; if (canSendStatus != SubscriptionStatus.subscribed || l2Lang == null || @@ -319,7 +326,9 @@ class Choreographer { itController.clear(); } - await (isRunningIT ? itController.getTranslationData(_useCustomInput) : igc.getIGCTextData()); + await (isRunningIT + ? itController.getTranslationData(_useCustomInput) + : igc.getIGCTextData()); } catch (err, stack) { ErrorHandler.logError( e: err, @@ -343,9 +352,12 @@ class Choreographer { void onITChoiceSelect(ITStep step) { _textController.setSystemText( _textController.text + step.continuances[step.chosen!].text, - step.continuances[step.chosen!].gold ? EditType.itGold : EditType.itStandard, + step.continuances[step.chosen!].gold + ? EditType.itGold + : EditType.itStandard, ); - _textController.selection = TextSelection.collapsed(offset: _textController.text.length); + _textController.selection = + TextSelection.collapsed(offset: _textController.text.length); _initChoreoRecord(); choreoRecord!.addRecord(_textController.text, step: step); @@ -393,11 +405,14 @@ class Choreographer { // return; // } - igc.igcTextData!.matches[matchIndex].match.choices![choiceIndex].selected = true; + igc.igcTextData!.matches[matchIndex].match.choices![choiceIndex] + .selected = true; - final isNormalizationError = l2Lang != null && igc.spanDataController.isNormalizationError(matchIndex, l2Lang!); + final isNormalizationError = l2Lang != null && + igc.spanDataController.isNormalizationError(matchIndex, l2Lang!); - final match = igc.igcTextData!.matches[matchIndex].copyWith..status = PangeaMatchStatus.accepted; + final match = igc.igcTextData!.matches[matchIndex].copyWith + ..status = PangeaMatchStatus.accepted; igc.igcTextData!.acceptReplacement( matchIndex, @@ -467,7 +482,8 @@ class Choreographer { void acceptNormalizationMatches() { final List indices = []; for (int i = 0; i < igc.igcTextData!.matches.length; i++) { - final isNormalizationError = l2Lang != null && igc.spanDataController.isNormalizationError(i, l2Lang!); + final isNormalizationError = l2Lang != null && + igc.spanDataController.isNormalizationError(i, l2Lang!); if (isNormalizationError) indices.add(i); } @@ -491,7 +507,11 @@ class Choreographer { final newMatch = match.copyWith; newMatch.status = PangeaMatchStatus.automatic; - newMatch.match.length = match.match.choices!.firstWhere((c) => c.isBestCorrection).value.characters.length; + newMatch.match.length = match.match.choices! + .firstWhere((c) => c.isBestCorrection) + .value + .characters + .length; _textController.setSystemText( igc.igcTextData!.originalInput, @@ -525,7 +545,8 @@ class Choreographer { igc.onIgnoreMatch(igc.igcTextData!.matches[matchIndex]); igc.igcTextData!.matches[matchIndex].status = PangeaMatchStatus.ignored; - final isNormalizationError = l2Lang != null && igc.spanDataController.isNormalizationError(matchIndex, l2Lang!); + final isNormalizationError = l2Lang != null && + igc.spanDataController.isNormalizationError(matchIndex, l2Lang!); if (!isNormalizationError) { _initChoreoRecord(); @@ -602,15 +623,18 @@ class Choreographer { String? get l2LangCode => l2Lang?.langCode; - LanguageModel? get l1Lang => pangeaController.languageController.activeL1Model(); + LanguageModel? get l1Lang => + pangeaController.languageController.activeL1Model(); String? get l1LangCode => l1Lang?.langCode; String? get userId => pangeaController.userController.userId; - bool get _noChange => _lastChecked != null && _lastChecked == _textController.text; + bool get _noChange => + _lastChecked != null && _lastChecked == _textController.text; - bool get isRunningIT => choreoMode == ChoreoMode.it && !itController.isTranslationDone; + bool get isRunningIT => + choreoMode == ChoreoMode.it && !itController.isTranslationDone; void startLoading() { _lastChecked = _textController.text; @@ -652,15 +676,18 @@ class Choreographer { _currentAssistanceState = assistanceState; } - LayerLinkAndKey get itBarLinkAndKey => MatrixState.pAnyState.layerLinkAndKey(itBarTransformTargetKey); + LayerLinkAndKey get itBarLinkAndKey => + MatrixState.pAnyState.layerLinkAndKey(itBarTransformTargetKey); String get itBarTransformTargetKey => 'it_bar$roomId'; - LayerLinkAndKey get inputLayerLinkAndKey => MatrixState.pAnyState.layerLinkAndKey(inputTransformTargetKey); + LayerLinkAndKey get inputLayerLinkAndKey => + MatrixState.pAnyState.layerLinkAndKey(inputTransformTargetKey); String get inputTransformTargetKey => 'input$roomId'; - LayerLinkAndKey get itBotLayerLinkAndKey => MatrixState.pAnyState.layerLinkAndKey(itBotTransformTargetKey); + LayerLinkAndKey get itBotLayerLinkAndKey => + MatrixState.pAnyState.layerLinkAndKey(itBotTransformTargetKey); String get itBotTransformTargetKey => 'itBot$roomId'; @@ -674,7 +701,8 @@ class Choreographer { chatController.room, ); - bool get isAutoIGCEnabled => pangeaController.permissionsController.isToolEnabled( + bool get isAutoIGCEnabled => + pangeaController.permissionsController.isToolEnabled( ToolSetting.autoIGC, chatController.room, ); @@ -706,7 +734,10 @@ class Choreographer { bool get canSendMessage { // if there's an error, let them send. we don't want to block them from sending in this case - if (errorService.isError || l2Lang == null || l1Lang == null || _timesClicked > 1) { + if (errorService.isError || + l2Lang == null || + l1Lang == null || + _timesClicked > 1) { return true; } @@ -725,8 +756,10 @@ class Choreographer { } // if they have relevant matches, don't let them send - final hasITMatches = igc.igcTextData!.matches.any((match) => match.isITStart); - final hasIGCMatches = igc.igcTextData!.matches.any((match) => !match.isITStart); + final hasITMatches = + igc.igcTextData!.matches.any((match) => match.isITStart); + final hasIGCMatches = + igc.igcTextData!.matches.any((match) => !match.isITStart); if ((itEnabled && hasITMatches) || (igcEnabled && hasIGCMatches)) { return false; } diff --git a/lib/pangea/choreographer/controllers/igc_controller.dart b/lib/pangea/choreographer/controllers/igc_controller.dart index c9eec7e35..63bd646bc 100644 --- a/lib/pangea/choreographer/controllers/igc_controller.dart +++ b/lib/pangea/choreographer/controllers/igc_controller.dart @@ -1,6 +1,12 @@ import 'dart:async'; import 'dart:developer'; +import 'package:flutter/foundation.dart'; +import 'package:flutter/material.dart'; + +import 'package:matrix/matrix.dart'; +import 'package:sentry_flutter/sentry_flutter.dart'; + import 'package:fluffychat/pangea/choreographer/controllers/choreographer.dart'; import 'package:fluffychat/pangea/choreographer/controllers/error_service.dart'; import 'package:fluffychat/pangea/choreographer/controllers/span_data_controller.dart'; @@ -10,11 +16,6 @@ import 'package:fluffychat/pangea/choreographer/repo/igc_repo.dart'; import 'package:fluffychat/pangea/choreographer/widgets/igc/span_card.dart'; import 'package:fluffychat/pangea/events/event_wrappers/pangea_message_event.dart'; import 'package:fluffychat/widgets/matrix.dart'; -import 'package:flutter/foundation.dart'; -import 'package:flutter/material.dart'; -import 'package:matrix/matrix.dart'; -import 'package:sentry_flutter/sentry_flutter.dart'; - import '../../common/utils/error_handler.dart'; import '../../common/utils/overlay.dart'; @@ -81,8 +82,10 @@ class IgcController { userId: choreographer.pangeaController.userController.userId!, userL1: choreographer.l1LangCode!, userL2: choreographer.l2LangCode!, - enableIGC: choreographer.igcEnabled && choreographer.choreoMode != ChoreoMode.it, - enableIT: choreographer.itEnabled && choreographer.choreoMode != ChoreoMode.it, + enableIGC: choreographer.igcEnabled && + choreographer.choreoMode != ChoreoMode.it, + enableIT: choreographer.itEnabled && + choreographer.choreoMode != ChoreoMode.it, prevMessages: _prevMessages(), ); @@ -101,10 +104,13 @@ class IgcController { } final IGCTextData igcTextDataResponse = - await _igcTextDataCache[reqBody.hashCode]!.data.timeout((const Duration(seconds: 10))); + await _igcTextDataCache[reqBody.hashCode]! + .data + .timeout((const Duration(seconds: 10))); // this will happen when the user changes the input while igc is fetching results - if (igcTextDataResponse.originalInput.trim() != choreographer.currentText.trim()) { + if (igcTextDataResponse.originalInput.trim() != + choreographer.currentText.trim()) { return; } // get ignored matches from the original igcTextData @@ -120,7 +126,8 @@ class IgcController { final List filteredMatches = List.from(igcTextData!.matches); for (final PangeaMatch match in igcTextData!.matches) { - final _IgnoredMatchCacheItem cacheEntry = _IgnoredMatchCacheItem(match: match); + final _IgnoredMatchCacheItem cacheEntry = + _IgnoredMatchCacheItem(match: match); if (_ignoredMatchCache.containsKey(cacheEntry.hashCode)) { filteredMatches.remove(match); @@ -139,7 +146,8 @@ class IgcController { // This will make the loading of span details faster for the user if (igcTextData?.matches.isNotEmpty ?? false) { for (int i = 0; i < igcTextData!.matches.length; i++) { - if (!igcTextData!.matches[i].isITStart && choreographer.l2Lang != null) { + if (!igcTextData!.matches[i].isITStart && + choreographer.l2Lang != null) { spanDataController.getSpanDetails(i, choreographer.l2Lang!); } } @@ -162,7 +170,8 @@ class IgcController { "itEnabled": choreographer.itEnabled, "matches": igcTextData?.matches.map((e) => e.toJson()), }, - level: err is TimeoutException ? SentryLevel.warning : SentryLevel.error, + level: + err is TimeoutException ? SentryLevel.warning : SentryLevel.error, ); clear(); } @@ -225,7 +234,8 @@ class IgcController { .where( (e) => e.type == EventTypes.Message && - (e.messageType == MessageTypes.Text || e.messageType == MessageTypes.Audio), + (e.messageType == MessageTypes.Text || + e.messageType == MessageTypes.Audio), ) .toList(); @@ -236,7 +246,8 @@ class IgcController { : PangeaMessageEvent( event: event, timeline: choreographer.chatController.timeline!, - ownMessage: event.senderId == choreographer.pangeaController.matrixState.client.userID, + ownMessage: event.senderId == + choreographer.pangeaController.matrixState.client.userID, ).getSpeechToTextLocal()?.transcript.text.trim(); // trim whitespace if (content == null) continue; messages.add( diff --git a/lib/pangea/choreographer/controllers/span_data_controller.dart b/lib/pangea/choreographer/controllers/span_data_controller.dart index d5ca4e64c..550f8f1f8 100644 --- a/lib/pangea/choreographer/controllers/span_data_controller.dart +++ b/lib/pangea/choreographer/controllers/span_data_controller.dart @@ -1,14 +1,16 @@ import 'dart:async'; import 'dart:developer'; +import 'package:flutter/foundation.dart'; + import 'package:collection/collection.dart'; + import 'package:fluffychat/pangea/choreographer/controllers/choreographer.dart'; import 'package:fluffychat/pangea/choreographer/models/span_data.dart'; import 'package:fluffychat/pangea/choreographer/repo/span_data_repo.dart'; import 'package:fluffychat/pangea/choreographer/utils/normalize_text.dart'; import 'package:fluffychat/pangea/common/utils/error_handler.dart'; import 'package:fluffychat/pangea/learning_settings/models/language_model.dart'; -import 'package:flutter/foundation.dart'; class _SpanDetailsCacheItem { Future data; @@ -69,7 +71,8 @@ class SpanDataController { ); return correctChoice != null && - normalizeString(correctChoice, spanLanguage.langCode) == normalizeString(errorSpan, spanLanguage.langCode); + normalizeString(correctChoice, spanLanguage.langCode) == + normalizeString(errorSpan, spanLanguage.langCode); } Future getSpanDetails( @@ -78,7 +81,8 @@ class SpanDataController { bool force = false, }) async { final SpanData? span = _getSpan(matchIndex); - if (span == null || (isNormalizationError(matchIndex, spanLanguage) && !force)) return; + if (span == null || + (isNormalizationError(matchIndex, spanLanguage) && !force)) return; final req = SpanDetailsRepoReqAndRes( userL1: choreographer.l1LangCode!, @@ -109,7 +113,8 @@ class SpanDataController { } try { - choreographer.igc.igcTextData!.matches[matchIndex].match = (await response).span; + choreographer.igc.igcTextData!.matches[matchIndex].match = + (await response).span; } catch (err, s) { ErrorHandler.logError(e: err, s: s, data: req.toJson()); _cache.remove(cacheKey); diff --git a/lib/pangea/choreographer/utils/normalize_text.dart b/lib/pangea/choreographer/utils/normalize_text.dart index 762ce969f..8ed13d9b3 100644 --- a/lib/pangea/choreographer/utils/normalize_text.dart +++ b/lib/pangea/choreographer/utils/normalize_text.dart @@ -1,7 +1,8 @@ import 'package:diacritic/diacritic.dart'; -import 'package:fluffychat/pangea/common/utils/error_handler.dart'; import 'package:test/test.dart'; +import 'package:fluffychat/pangea/common/utils/error_handler.dart'; + // The intention of this function is to normalize text for comparison purposes. // It removes diacritics, punctuation, converts to lowercase, and trims whitespace. // We would like esta = está, hello! = Hello, etc. @@ -16,11 +17,13 @@ String normalizeString(String input, String languageCode) { normalized = _applyLanguageSpecificNormalization(normalized, languageCode); // Step 3: Replace hyphens and other dash-like characters with spaces - normalized = normalized.replaceAll(RegExp(r'[-\u2010-\u2015\u2212\uFE58\uFE63\uFF0D]'), ' '); + normalized = normalized.replaceAll( + RegExp(r'[-\u2010-\u2015\u2212\uFE58\uFE63\uFF0D]'), ' '); // Step 4: Remove punctuation (including Unicode punctuation) // This removes ASCII and Unicode punctuation while preserving letters, numbers, and spaces - normalized = normalized.replaceAll(RegExp(r'[\p{P}\p{S}]', unicode: true), ''); + normalized = + normalized.replaceAll(RegExp(r'[\p{P}\p{S}]', unicode: true), ''); // Step 5: Normalize whitespace (collapse multiple spaces, trim) normalized = normalized.replaceAll(RegExp(r'\s+'), ' ').trim(); @@ -431,11 +434,13 @@ void runNormalizationTests() { passed++; print('✓ Test ${i + 1} PASSED: "$input" → "$actual"'); } else { - print('✗ Test ${i + 1} FAILED: "$input" → "$actual" (expected: "$expected")'); + print( + '✗ Test ${i + 1} FAILED: "$input" → "$actual" (expected: "$expected")'); } } - print('\nTest Results: $passed/$total tests passed (${(passed / total * 100).toStringAsFixed(1)}%)'); + print( + '\nTest Results: $passed/$total tests passed (${(passed / total * 100).toStringAsFixed(1)}%)'); } // Main function to run the tests when executed directly @@ -448,7 +453,8 @@ void main() { final expected = testCase['expected']!; test('Test ${i + 1}: "$input" should normalize to "$expected"', () { - final actual = normalizeString(input, 'en'); // Default to English for tests + final actual = + normalizeString(input, 'en'); // Default to English for tests expect( actual, equals(expected), diff --git a/lib/pangea/choreographer/widgets/igc/span_card.dart b/lib/pangea/choreographer/widgets/igc/span_card.dart index 4f9948eb2..e90bda30b 100644 --- a/lib/pangea/choreographer/widgets/igc/span_card.dart +++ b/lib/pangea/choreographer/widgets/igc/span_card.dart @@ -1,3 +1,5 @@ +import 'package:flutter/material.dart'; + import 'package:fluffychat/l10n/l10n.dart'; import 'package:fluffychat/pangea/bot/utils/bot_style.dart'; import 'package:fluffychat/pangea/choreographer/controllers/choreographer.dart'; @@ -7,8 +9,6 @@ import 'package:fluffychat/pangea/choreographer/models/pangea_match_model.dart'; import 'package:fluffychat/pangea/choreographer/models/span_data.dart'; import 'package:fluffychat/pangea/common/utils/error_handler.dart'; import 'package:fluffychat/pangea/toolbar/controllers/tts_controller.dart'; -import 'package:flutter/material.dart'; - import '../../../../widgets/matrix.dart'; import '../../../bot/widgets/bot_face_svg.dart'; import '../choice_array.dart'; @@ -54,7 +54,8 @@ class SpanCardState extends State { PangeaMatch? get pangeaMatch { if (widget.choreographer.igc.igcTextData == null) return null; - if (widget.matchIndex >= widget.choreographer.igc.igcTextData!.matches.length) { + if (widget.matchIndex >= + widget.choreographer.igc.igcTextData!.matches.length) { ErrorHandler.logError( m: "matchIndex out of bounds in span card", data: { @@ -74,7 +75,8 @@ class SpanCardState extends State { } SpanChoice? _choiceByIndex(int index) { - if (pangeaMatch?.match.choices == null || pangeaMatch!.match.choices!.length <= index) { + if (pangeaMatch?.match.choices == null || + pangeaMatch!.match.choices!.length <= index) { return null; } return pangeaMatch?.match.choices?[index]; @@ -86,7 +88,8 @@ class SpanCardState extends State { } // if user ever selected the correct choice, automatically select it - final selectedCorrectIndex = pangeaMatch!.match.choices!.indexWhere((choice) { + final selectedCorrectIndex = + pangeaMatch!.match.choices!.indexWhere((choice) { return choice.selected && choice.isBestCorrection; }); @@ -100,7 +103,8 @@ class SpanCardState extends State { final numChoices = pangeaMatch!.match.choices!.length; for (int i = 0; i < numChoices; i++) { final choice = _choiceByIndex(i); - if (choice!.timestamp != null && (mostRecent == null || choice.timestamp!.isAfter(mostRecent))) { + if (choice!.timestamp != null && + (mostRecent == null || choice.timestamp!.isAfter(mostRecent))) { mostRecent = choice.timestamp; selectedChoiceIndex = i; } @@ -151,7 +155,9 @@ class SpanCardState extends State { selectedChoice!.timestamp = DateTime.now(); selectedChoice!.selected = true; setState( - () => (selectedChoice!.isBestCorrection ? BotExpression.gold : BotExpression.surprised), + () => (selectedChoice!.isBestCorrection + ? BotExpression.gold + : BotExpression.surprised), ); } } @@ -177,7 +183,8 @@ class SpanCardState extends State { } void _showFirstMatch() { - if (widget.choreographer.igc.igcTextData != null && widget.choreographer.igc.igcTextData!.matches.isNotEmpty) { + if (widget.choreographer.igc.igcTextData != null && + widget.choreographer.igc.igcTextData!.matches.isNotEmpty) { widget.choreographer.igc.showFirstMatch(context); } else { MatrixState.pAnyState.closeOverlay(); @@ -235,10 +242,12 @@ class WordMatchContent extends StatelessWidget { ), ) .toList(), - onPressed: (value, index) => controller._onChoiceSelect(index), + onPressed: (value, index) => + controller._onChoiceSelect(index), selectedChoiceIndex: controller.selectedChoiceIndex, id: controller.pangeaMatch!.hashCode.toString(), - langCode: MatrixState.pangeaController.languageController.activeL2Code(), + langCode: MatrixState.pangeaController.languageController + .activeL2Code(), ), const SizedBox(height: 12), PromptAndFeedback(controller: controller), @@ -275,7 +284,9 @@ class WordMatchContent extends StatelessWidget { child: Opacity( opacity: controller.selectedChoiceIndex != null ? 1.0 : 0.5, child: TextButton( - onPressed: controller.selectedChoiceIndex != null ? controller._onReplaceSelected : null, + onPressed: controller.selectedChoiceIndex != null + ? controller._onReplaceSelected + : null, style: ButtonStyle( backgroundColor: WidgetStateProperty.all( (controller.selectedChoice != null @@ -322,7 +333,9 @@ class PromptAndFeedback extends StatelessWidget { } return Container( - constraints: controller.pangeaMatch!.isITStart ? null : const BoxConstraints(minHeight: 75.0), + constraints: controller.pangeaMatch!.isITStart + ? null + : const BoxConstraints(minHeight: 75.0), child: Column( mainAxisAlignment: MainAxisAlignment.center, crossAxisAlignment: CrossAxisAlignment.center, @@ -352,9 +365,11 @@ class PromptAndFeedback extends StatelessWidget { loading: controller.fetchingData, ), ], - if (!controller.fetchingData && controller.selectedChoiceIndex == null) + if (!controller.fetchingData && + controller.selectedChoiceIndex == null) Text( - controller.pangeaMatch!.match.type.typeName.defaultPrompt(context), + controller.pangeaMatch!.match.type.typeName + .defaultPrompt(context), style: BotStyle.text(context).copyWith( fontStyle: FontStyle.italic, ), From 586c9613c13247d61a234bb42756dfcbd5793937 Mon Sep 17 00:00:00 2001 From: ggurdin Date: Fri, 7 Nov 2025 09:10:54 -0500 Subject: [PATCH 03/11] move unnecessary references to L2 into helper function, move text normalization tests into their own file --- .../controllers/choreographer.dart | 12 +- .../controllers/igc_controller.dart | 5 +- .../controllers/span_data_controller.dart | 17 +- .../choreographer/utils/normalize_text.dart | 352 +----------------- .../choreographer/widgets/igc/span_card.dart | 13 - test/pangea/text_normalization_test.dart | 352 ++++++++++++++++++ 6 files changed, 375 insertions(+), 376 deletions(-) create mode 100644 test/pangea/text_normalization_test.dart diff --git a/lib/pangea/choreographer/controllers/choreographer.dart b/lib/pangea/choreographer/controllers/choreographer.dart index 18a6c1f58..5e1435ae5 100644 --- a/lib/pangea/choreographer/controllers/choreographer.dart +++ b/lib/pangea/choreographer/controllers/choreographer.dart @@ -408,8 +408,8 @@ class Choreographer { igc.igcTextData!.matches[matchIndex].match.choices![choiceIndex] .selected = true; - final isNormalizationError = l2Lang != null && - igc.spanDataController.isNormalizationError(matchIndex, l2Lang!); + final isNormalizationError = + igc.spanDataController.isL2NormalizationError(matchIndex); final match = igc.igcTextData!.matches[matchIndex].copyWith ..status = PangeaMatchStatus.accepted; @@ -482,8 +482,8 @@ class Choreographer { void acceptNormalizationMatches() { final List indices = []; for (int i = 0; i < igc.igcTextData!.matches.length; i++) { - final isNormalizationError = l2Lang != null && - igc.spanDataController.isNormalizationError(i, l2Lang!); + final isNormalizationError = + igc.spanDataController.isL2NormalizationError(i); if (isNormalizationError) indices.add(i); } @@ -545,8 +545,8 @@ class Choreographer { igc.onIgnoreMatch(igc.igcTextData!.matches[matchIndex]); igc.igcTextData!.matches[matchIndex].status = PangeaMatchStatus.ignored; - final isNormalizationError = l2Lang != null && - igc.spanDataController.isNormalizationError(matchIndex, l2Lang!); + final isNormalizationError = + igc.spanDataController.isL2NormalizationError(matchIndex); if (!isNormalizationError) { _initChoreoRecord(); diff --git a/lib/pangea/choreographer/controllers/igc_controller.dart b/lib/pangea/choreographer/controllers/igc_controller.dart index 63bd646bc..abefff19b 100644 --- a/lib/pangea/choreographer/controllers/igc_controller.dart +++ b/lib/pangea/choreographer/controllers/igc_controller.dart @@ -146,9 +146,8 @@ class IgcController { // This will make the loading of span details faster for the user if (igcTextData?.matches.isNotEmpty ?? false) { for (int i = 0; i < igcTextData!.matches.length; i++) { - if (!igcTextData!.matches[i].isITStart && - choreographer.l2Lang != null) { - spanDataController.getSpanDetails(i, choreographer.l2Lang!); + if (!igcTextData!.matches[i].isITStart) { + spanDataController.getSpanDetails(i); } } } diff --git a/lib/pangea/choreographer/controllers/span_data_controller.dart b/lib/pangea/choreographer/controllers/span_data_controller.dart index 550f8f1f8..f2628230a 100644 --- a/lib/pangea/choreographer/controllers/span_data_controller.dart +++ b/lib/pangea/choreographer/controllers/span_data_controller.dart @@ -11,6 +11,7 @@ import 'package:fluffychat/pangea/choreographer/repo/span_data_repo.dart'; import 'package:fluffychat/pangea/choreographer/utils/normalize_text.dart'; import 'package:fluffychat/pangea/common/utils/error_handler.dart'; import 'package:fluffychat/pangea/learning_settings/models/language_model.dart'; +import 'package:fluffychat/widgets/matrix.dart'; class _SpanDetailsCacheItem { Future data; @@ -55,7 +56,13 @@ class SpanDataController { return choreographer.igc.igcTextData!.matches[matchIndex].match; } - bool isNormalizationError(int matchIndex, LanguageModel spanLanguage) { + bool isL2NormalizationError(int matchIndex) { + final l2 = MatrixState.pangeaController.languageController.userL2; + if (l2 == null) return false; + return _isNormalizationError(matchIndex, l2); + } + + bool _isNormalizationError(int matchIndex, LanguageModel spanLanguage) { final span = _getSpan(matchIndex); if (span == null) return false; @@ -76,13 +83,13 @@ class SpanDataController { } Future getSpanDetails( - int matchIndex, - LanguageModel spanLanguage, { + int matchIndex, { bool force = false, }) async { final SpanData? span = _getSpan(matchIndex); - if (span == null || - (isNormalizationError(matchIndex, spanLanguage) && !force)) return; + if (span == null || (isL2NormalizationError(matchIndex) && !force)) { + return; + } final req = SpanDetailsRepoReqAndRes( userL1: choreographer.l1LangCode!, diff --git a/lib/pangea/choreographer/utils/normalize_text.dart b/lib/pangea/choreographer/utils/normalize_text.dart index 8ed13d9b3..96eee9422 100644 --- a/lib/pangea/choreographer/utils/normalize_text.dart +++ b/lib/pangea/choreographer/utils/normalize_text.dart @@ -1,5 +1,4 @@ import 'package:diacritic/diacritic.dart'; -import 'package:test/test.dart'; import 'package:fluffychat/pangea/common/utils/error_handler.dart'; @@ -18,7 +17,9 @@ String normalizeString(String input, String languageCode) { // Step 3: Replace hyphens and other dash-like characters with spaces normalized = normalized.replaceAll( - RegExp(r'[-\u2010-\u2015\u2212\uFE58\uFE63\uFF0D]'), ' '); + RegExp(r'[-\u2010-\u2015\u2212\uFE58\uFE63\uFF0D]'), + ' ', + ); // Step 4: Remove punctuation (including Unicode punctuation) // This removes ASCII and Unicode punctuation while preserving letters, numbers, and spaces @@ -116,351 +117,4 @@ String _removeGreekAccents(String text) { .replaceAll('Ό', 'Ο') .replaceAll('Ύ', 'Υ') .replaceAll('Ώ', 'Ω'); -} // Comprehensive test cases for the normalizeString function - -// Covers all 49 supported languages with various edge cases -final List> normalizeTestCases = [ - // 1. Amharic (am) - beta - {"input": "ሰላም!", "expected": "ሰላም"}, - {"input": "ተማሪ።", "expected": "ተማሪ"}, - {"input": "ኢትዮጵያ...", "expected": "ኢትዮጵያ"}, - - // 2. Arabic (ar) - beta - {"input": "السلام عليكم!", "expected": "السلام عليكم"}, - {"input": "مرحباً", "expected": "مرحباً"}, - {"input": "القاهرة.", "expected": "القاهرة"}, - {"input": "مدرسة؟", "expected": "مدرسة"}, - - // 3. Bengali (bn) - beta - {"input": "নমস্কার!", "expected": "নমস্কার"}, - {"input": "ভালো আছেন?", "expected": "ভালো আছেন"}, - {"input": "ঢাকা।", "expected": "ঢাকা"}, - - // 4. Bulgarian (bg) - beta - {"input": "Здравей!", "expected": "здравей"}, - {"input": "България", "expected": "българия"}, - {"input": "София.", "expected": "софия"}, - - // 5. Catalan (ca) - full - {"input": "Hola!", "expected": "hola"}, - {"input": "França", "expected": "franca"}, - {"input": "Barcelòna...", "expected": "barcelòna"}, - {"input": "això", "expected": "això"}, - - // 6. Czech (cs) - beta - {"input": "Dobrý den!", "expected": "dobry den"}, - {"input": "Děkuji", "expected": "dekuji"}, - {"input": "Praha.", "expected": "praha"}, - {"input": "škola?", "expected": "skola"}, - - // 7. Danish (da) - beta - {"input": "Hej!", "expected": "hej"}, - {"input": "København", "expected": "kobenhavn"}, - {"input": "Danskе.", "expected": "danske"}, - {"input": "æøå", "expected": "æøå"}, - - // 8. German (de) - full - {"input": "Guten Tag!", "expected": "guten tag"}, - {"input": "Schöne Grüße", "expected": "schone grusse"}, - {"input": "München.", "expected": "munchen"}, - {"input": "Straße?", "expected": "strasse"}, - {"input": "Hörst du mich?", "expected": "horst du mich"}, - - // 9. Greek (el) - beta - {"input": "Γεια σας!", "expected": "γεια σας"}, - {"input": "Αθήνα", "expected": "αθηνα"}, - {"input": "ελληνικά.", "expected": "ελληνικα"}, - - // 10. English (en) - full - {"input": "Hello world!", "expected": "hello world"}, - {"input": "It's a beautiful day.", "expected": "its a beautiful day"}, - {"input": "Don't worry, be happy!", "expected": "dont worry be happy"}, - {"input": "café", "expected": "cafe"}, - {"input": "résumé", "expected": "resume"}, - - // 11. Spanish (es) - full - {"input": "¡Hola mundo!", "expected": "hola mundo"}, - {"input": "Adiós", "expected": "adios"}, - {"input": "España.", "expected": "espana"}, - {"input": "niño", "expected": "nino"}, - {"input": "¿Cómo estás?", "expected": "como estas"}, - - // 12. Estonian (et) - beta - {"input": "Tere!", "expected": "tere"}, - {"input": "Tallinn", "expected": "tallinn"}, - {"input": "Eesti.", "expected": "eesti"}, - - // 13. Basque (eu) - beta - {"input": "Kaixo!", "expected": "kaixo"}, - {"input": "Euskera", "expected": "euskera"}, - {"input": "Bilbo.", "expected": "bilbo"}, - - // 14. Finnish (fi) - beta - {"input": "Hei!", "expected": "hei"}, - {"input": "Helsinki", "expected": "helsinki"}, - {"input": "Suomi.", "expected": "suomi"}, - {"input": "Käännös", "expected": "kaannos"}, - - // 15. French (fr) - full - {"input": "Bonjour!", "expected": "bonjour"}, - {"input": "À bientôt", "expected": "a bientot"}, - {"input": "Paris.", "expected": "paris"}, - {"input": "Français?", "expected": "francais"}, - {"input": "C'est magnifique!", "expected": "cest magnifique"}, - - // 16. Galician (gl) - beta - {"input": "Ola!", "expected": "ola"}, - {"input": "Galicia", "expected": "galicia"}, - {"input": "Santiago.", "expected": "santiago"}, - - // 17. Gujarati (gu) - beta - {"input": "નમસ્તે!", "expected": "નમસ્તે"}, - {"input": "ગુજરાત", "expected": "ગુજરાત"}, - {"input": "અમદાવાદ.", "expected": "અમદાવાદ"}, - - // 18. Hindi (hi) - beta - {"input": "नमस्ते!", "expected": "नमस्ते"}, - {"input": "भारत", "expected": "भारत"}, - {"input": "दिल्ली.", "expected": "दिल्ली"}, - {"input": "शिक्षा?", "expected": "शिक्षा"}, - - // 19. Hungarian (hu) - beta - {"input": "Szia!", "expected": "szia"}, - {"input": "Budapest", "expected": "budapest"}, - {"input": "Magyar.", "expected": "magyar"}, - {"input": "köszönöm", "expected": "koszonom"}, - - // 20. Indonesian (id) - beta - {"input": "Halo!", "expected": "halo"}, - {"input": "Jakarta", "expected": "jakarta"}, - {"input": "Indonesia.", "expected": "indonesia"}, - {"input": "selamat pagi", "expected": "selamat pagi"}, - - // 21. Italian (it) - full - {"input": "Ciao!", "expected": "ciao"}, - {"input": "Arrivederci", "expected": "arrivederci"}, - {"input": "Roma.", "expected": "roma"}, - {"input": "perché?", "expected": "perche"}, - {"input": "È bellissimo!", "expected": "e bellissimo"}, - - // 22. Japanese (ja) - full - {"input": "こんにちは！", "expected": "こんにちは"}, - {"input": "東京", "expected": "東京"}, - {"input": "ありがとう。", "expected": "ありがとう"}, - {"input": "さようなら？", "expected": "さようなら"}, - - // 23. Kannada (kn) - beta - {"input": "ನಮಸ್ತೆ!", "expected": "ನಮಸ್ತೆ"}, - {"input": "ಬೆಂಗಳೂರು", "expected": "ಬೆಂಗಳೂರು"}, - {"input": "ಕರ್ನಾಟಕ.", "expected": "ಕರ್ನಾಟಕ"}, - - // 24. Korean (ko) - full - {"input": "안녕하세요!", "expected": "안녕하세요"}, - {"input": "서울", "expected": "서울"}, - {"input": "한국어.", "expected": "한국어"}, - {"input": "감사합니다?", "expected": "감사합니다"}, - - // 25. Lithuanian (lt) - beta - {"input": "Labas!", "expected": "labas"}, - {"input": "Vilnius", "expected": "vilnius"}, - {"input": "Lietuva.", "expected": "lietuva"}, - {"input": "ačiū", "expected": "aciu"}, - - // 26. Latvian (lv) - beta - {"input": "Sveiki!", "expected": "sveiki"}, - {"input": "Rīga", "expected": "riga"}, - {"input": "Latvija.", "expected": "latvija"}, - - // 27. Malay (ms) - beta - {"input": "Selamat pagi!", "expected": "selamat pagi"}, - {"input": "Kuala Lumpur", "expected": "kuala lumpur"}, - {"input": "Malaysia.", "expected": "malaysia"}, - - // 28. Mongolian (mn) - beta - {"input": "Сайн байна уу!", "expected": "сайн байна уу"}, - {"input": "Улаанбаатар", "expected": "улаанбаатар"}, - {"input": "Монгол.", "expected": "монгол"}, - - // 29. Marathi (mr) - beta - {"input": "नमस्कार!", "expected": "नमस्कार"}, - {"input": "मुंबई", "expected": "मुंबई"}, - {"input": "महाराष्ट्र.", "expected": "महाराष्ट्र"}, - - // 30. Dutch (nl) - beta - {"input": "Hallo!", "expected": "hallo"}, - {"input": "Amsterdam", "expected": "amsterdam"}, - {"input": "Nederland.", "expected": "nederland"}, - {"input": "dankjewel", "expected": "dankjewel"}, - - // 31. Punjabi (pa) - beta - {"input": "ਸਤਿ ਸ਼੍ਰੀ ਅਕਾਲ!", "expected": "ਸਤਿ ਸ਼੍ਰੀ ਅਕਾਲ"}, - {"input": "ਪੰਜਾਬ", "expected": "ਪੰਜਾਬ"}, - {"input": "ਅੰਮ੍ਰਿਤਸਰ.", "expected": "ਅੰਮ੍ਰਿਤਸਰ"}, - - // 32. Polish (pl) - beta - {"input": "Cześć!", "expected": "czesc"}, - {"input": "Warszawa", "expected": "warszawa"}, - {"input": "Polska.", "expected": "polska"}, - {"input": "dziękuję", "expected": "dziekuje"}, - - // 33. Portuguese (pt) - full - {"input": "Olá!", "expected": "ola"}, - {"input": "Obrigado", "expected": "obrigado"}, - {"input": "São Paulo.", "expected": "sao paulo"}, - {"input": "coração", "expected": "coracao"}, - {"input": "não?", "expected": "nao"}, - - // 34. Romanian (ro) - beta - {"input": "Salut!", "expected": "salut"}, - {"input": "București", "expected": "bucuresti"}, - {"input": "România.", "expected": "romania"}, - {"input": "mulțumesc", "expected": "multumesc"}, - - // 35. Russian (ru) - full - {"input": "Привет!", "expected": "привет"}, - {"input": "Москва", "expected": "москва"}, - {"input": "Россия.", "expected": "россия"}, - {"input": "спасибо?", "expected": "спасибо"}, - {"input": "магазин", "expected": "магазин"}, - {"input": "магазин.", "expected": "магазин"}, - - // 36. Slovak (sk) - beta - {"input": "Ahoj!", "expected": "ahoj"}, - {"input": "Bratislava", "expected": "bratislava"}, - {"input": "Slovensko.", "expected": "slovensko"}, - {"input": "ďakujem", "expected": "dakujem"}, - - // 37. Serbian (sr) - beta - {"input": "Здраво!", "expected": "здраво"}, - {"input": "Београд", "expected": "београд"}, - {"input": "Србија.", "expected": "србија"}, - - // 38. Ukrainian (uk) - beta - {"input": "Привіт!", "expected": "привіт"}, - {"input": "Київ", "expected": "київ"}, - {"input": "Україна.", "expected": "україна"}, - - // 39. Urdu (ur) - beta - {"input": "السلام علیکم!", "expected": "السلام علیکم"}, - {"input": "کراچی", "expected": "کراچی"}, - {"input": "پاکستان.", "expected": "پاکستان"}, - - // 40. Vietnamese (vi) - full - {"input": "Xin chào!", "expected": "xin chao"}, - {"input": "Hà Nội", "expected": "ha noi"}, - {"input": "Việt Nam.", "expected": "viet nam"}, - {"input": "cảm ơn?", "expected": "cam on"}, - - // 41. Cantonese (yue) - beta - {"input": "你好！", "expected": "你好"}, - {"input": "香港", "expected": "香港"}, - {"input": "廣東話.", "expected": "廣東話"}, - - // 42. Chinese Simplified (zh-CN) - full - {"input": "你好！", "expected": "你好"}, - {"input": "北京", "expected": "北京"}, - {"input": "中国.", "expected": "中国"}, - {"input": "谢谢?", "expected": "谢谢"}, - - // 43. Chinese Traditional (zh-TW) - full - {"input": "您好！", "expected": "您好"}, - {"input": "台北", "expected": "台北"}, - {"input": "台灣.", "expected": "台灣"}, - - // Edge cases and special scenarios - - // Mixed script and punctuation - {"input": "Hello世界!", "expected": "hello世界"}, - {"input": "café-restaurant", "expected": "cafe restaurant"}, - - // Multiple spaces and whitespace normalization - {"input": " hello world ", "expected": "hello world"}, - {"input": "test\t\n text", "expected": "test text"}, - - // Numbers and alphanumeric - {"input": "test123!", "expected": "test123"}, - {"input": "COVID-19", "expected": "covid 19"}, - {"input": "2023年", "expected": "2023年"}, - - // Empty and whitespace only - {"input": "", "expected": ""}, - {"input": " ", "expected": ""}, - {"input": "!!!", "expected": ""}, - - // Special punctuation combinations - {"input": "What?!?", "expected": "what"}, - {"input": "Well...", "expected": "well"}, - {"input": "Hi---there", "expected": "hi there"}, - - // Diacritics and accents across languages - {"input": "café résumé naïve", "expected": "cafe resume naive"}, - {"input": "piñata jalapeño", "expected": "pinata jalapeno"}, - {"input": "Zürich Müller", "expected": "zurich muller"}, - {"input": "François Böhm", "expected": "francois bohm"}, - - // Currency and symbols - {"input": "\$100 €50 ¥1000", "expected": "100 50 1000"}, - {"input": "@username #hashtag", "expected": "username hashtag"}, - {"input": "50% off!", "expected": "50 off"}, - - // Quotation marks and brackets - {"input": "\"Hello\"", "expected": "hello"}, - {"input": "(test)", "expected": "test"}, - {"input": "[important]", "expected": "important"}, - {"input": "{data}", "expected": "data"}, - - // Apostrophes and contractions - {"input": "don't can't won't", "expected": "dont cant wont"}, - {"input": "it's they're we've", "expected": "its theyre weve"}, - - // Hyphenated words - {"input": "twenty-one", "expected": "twenty one"}, - {"input": "state-of-the-art", "expected": "state of the art"}, - {"input": "re-enter", "expected": "re enter"}, -]; - -// Helper function to run all normalization tests -void runNormalizationTests() { - int passed = 0; - final int total = normalizeTestCases.length; - - for (int i = 0; i < normalizeTestCases.length; i++) { - final testCase = normalizeTestCases[i]; - final input = testCase['input']!; - final expected = testCase['expected']!; - final actual = normalizeString(input, 'en'); // Default to English for tests - - if (actual == expected) { - passed++; - print('✓ Test ${i + 1} PASSED: "$input" → "$actual"'); - } else { - print( - '✗ Test ${i + 1} FAILED: "$input" → "$actual" (expected: "$expected")'); - } - } - - print( - '\nTest Results: $passed/$total tests passed (${(passed / total * 100).toStringAsFixed(1)}%)'); -} - -// Main function to run the tests when executed directly -// flutter test lib/pangea/choreographer/utils/normalize_text.dart -void main() { - group('Normalize String Tests', () { - for (int i = 0; i < normalizeTestCases.length; i++) { - final testCase = normalizeTestCases[i]; - final input = testCase['input']!; - final expected = testCase['expected']!; - - test('Test ${i + 1}: "$input" should normalize to "$expected"', () { - final actual = - normalizeString(input, 'en'); // Default to English for tests - expect( - actual, - equals(expected), - reason: 'Input: "$input" → Got: "$actual" → Expected: "$expected"', - ); - }); - } - }); } diff --git a/lib/pangea/choreographer/widgets/igc/span_card.dart b/lib/pangea/choreographer/widgets/igc/span_card.dart index e90bda30b..fb3a75fa6 100644 --- a/lib/pangea/choreographer/widgets/igc/span_card.dart +++ b/lib/pangea/choreographer/widgets/igc/span_card.dart @@ -120,21 +120,8 @@ class SpanCardState extends State { fetchingData = true; }); - if (widget.choreographer.l2Lang == null) { - ErrorHandler.logError( - m: "l2Lang is null when trying to get span details", - data: { - "matchIndex": widget.matchIndex, - }, - ); - setState(() { - fetchingData = false; - }); - return; - } await widget.choreographer.igc.spanDataController.getSpanDetails( widget.matchIndex, - widget.choreographer.l2Lang!, force: force, ); diff --git a/test/pangea/text_normalization_test.dart b/test/pangea/text_normalization_test.dart new file mode 100644 index 000000000..a0b141cd4 --- /dev/null +++ b/test/pangea/text_normalization_test.dart @@ -0,0 +1,352 @@ +import 'package:flutter_test/flutter_test.dart'; +import 'package:matrix/matrix_api_lite/utils/logs.dart'; + +import 'package:fluffychat/pangea/choreographer/utils/normalize_text.dart'; + +final List> normalizeTestCases = [ + // 1. Amharic (am) - beta + {"input": "ሰላም!", "expected": "ሰላም"}, + {"input": "ተማሪ።", "expected": "ተማሪ"}, + {"input": "ኢትዮጵያ...", "expected": "ኢትዮጵያ"}, + + // 2. Arabic (ar) - beta + {"input": "السلام عليكم!", "expected": "السلام عليكم"}, + {"input": "مرحباً", "expected": "مرحباً"}, + {"input": "القاهرة.", "expected": "القاهرة"}, + {"input": "مدرسة؟", "expected": "مدرسة"}, + + // 3. Bengali (bn) - beta + {"input": "নমস্কার!", "expected": "নমস্কার"}, + {"input": "ভালো আছেন?", "expected": "ভালো আছেন"}, + {"input": "ঢাকা।", "expected": "ঢাকা"}, + + // 4. Bulgarian (bg) - beta + {"input": "Здравей!", "expected": "здравей"}, + {"input": "България", "expected": "българия"}, + {"input": "София.", "expected": "софия"}, + + // 5. Catalan (ca) - full + {"input": "Hola!", "expected": "hola"}, + {"input": "França", "expected": "franca"}, + {"input": "Barcelòna...", "expected": "barcelòna"}, + {"input": "això", "expected": "això"}, + + // 6. Czech (cs) - beta + {"input": "Dobrý den!", "expected": "dobry den"}, + {"input": "Děkuji", "expected": "dekuji"}, + {"input": "Praha.", "expected": "praha"}, + {"input": "škola?", "expected": "skola"}, + + // 7. Danish (da) - beta + {"input": "Hej!", "expected": "hej"}, + {"input": "København", "expected": "kobenhavn"}, + {"input": "Danskе.", "expected": "danske"}, + {"input": "æøå", "expected": "æøå"}, + + // 8. German (de) - full + {"input": "Guten Tag!", "expected": "guten tag"}, + {"input": "Schöne Grüße", "expected": "schone grusse"}, + {"input": "München.", "expected": "munchen"}, + {"input": "Straße?", "expected": "strasse"}, + {"input": "Hörst du mich?", "expected": "horst du mich"}, + + // 9. Greek (el) - beta + {"input": "Γεια σας!", "expected": "γεια σας"}, + {"input": "Αθήνα", "expected": "αθηνα"}, + {"input": "ελληνικά.", "expected": "ελληνικα"}, + + // 10. English (en) - full + {"input": "Hello world!", "expected": "hello world"}, + {"input": "It's a beautiful day.", "expected": "its a beautiful day"}, + {"input": "Don't worry, be happy!", "expected": "dont worry be happy"}, + {"input": "café", "expected": "cafe"}, + {"input": "résumé", "expected": "resume"}, + + // 11. Spanish (es) - full + {"input": "¡Hola mundo!", "expected": "hola mundo"}, + {"input": "Adiós", "expected": "adios"}, + {"input": "España.", "expected": "espana"}, + {"input": "niño", "expected": "nino"}, + {"input": "¿Cómo estás?", "expected": "como estas"}, + + // 12. Estonian (et) - beta + {"input": "Tere!", "expected": "tere"}, + {"input": "Tallinn", "expected": "tallinn"}, + {"input": "Eesti.", "expected": "eesti"}, + + // 13. Basque (eu) - beta + {"input": "Kaixo!", "expected": "kaixo"}, + {"input": "Euskera", "expected": "euskera"}, + {"input": "Bilbo.", "expected": "bilbo"}, + + // 14. Finnish (fi) - beta + {"input": "Hei!", "expected": "hei"}, + {"input": "Helsinki", "expected": "helsinki"}, + {"input": "Suomi.", "expected": "suomi"}, + {"input": "Käännös", "expected": "kaannos"}, + + // 15. French (fr) - full + {"input": "Bonjour!", "expected": "bonjour"}, + {"input": "À bientôt", "expected": "a bientot"}, + {"input": "Paris.", "expected": "paris"}, + {"input": "Français?", "expected": "francais"}, + {"input": "C'est magnifique!", "expected": "cest magnifique"}, + + // 16. Galician (gl) - beta + {"input": "Ola!", "expected": "ola"}, + {"input": "Galicia", "expected": "galicia"}, + {"input": "Santiago.", "expected": "santiago"}, + + // 17. Gujarati (gu) - beta + {"input": "નમસ્તે!", "expected": "નમસ્તે"}, + {"input": "ગુજરાત", "expected": "ગુજરાત"}, + {"input": "અમદાવાદ.", "expected": "અમદાવાદ"}, + + // 18. Hindi (hi) - beta + {"input": "नमस्ते!", "expected": "नमस्ते"}, + {"input": "भारत", "expected": "भारत"}, + {"input": "दिल्ली.", "expected": "दिल्ली"}, + {"input": "शिक्षा?", "expected": "शिक्षा"}, + + // 19. Hungarian (hu) - beta + {"input": "Szia!", "expected": "szia"}, + {"input": "Budapest", "expected": "budapest"}, + {"input": "Magyar.", "expected": "magyar"}, + {"input": "köszönöm", "expected": "koszonom"}, + + // 20. Indonesian (id) - beta + {"input": "Halo!", "expected": "halo"}, + {"input": "Jakarta", "expected": "jakarta"}, + {"input": "Indonesia.", "expected": "indonesia"}, + {"input": "selamat pagi", "expected": "selamat pagi"}, + + // 21. Italian (it) - full + {"input": "Ciao!", "expected": "ciao"}, + {"input": "Arrivederci", "expected": "arrivederci"}, + {"input": "Roma.", "expected": "roma"}, + {"input": "perché?", "expected": "perche"}, + {"input": "È bellissimo!", "expected": "e bellissimo"}, + + // 22. Japanese (ja) - full + {"input": "こんにちは！", "expected": "こんにちは"}, + {"input": "東京", "expected": "東京"}, + {"input": "ありがとう。", "expected": "ありがとう"}, + {"input": "さようなら？", "expected": "さようなら"}, + + // 23. Kannada (kn) - beta + {"input": "ನಮಸ್ತೆ!", "expected": "ನಮಸ್ತೆ"}, + {"input": "ಬೆಂಗಳೂರು", "expected": "ಬೆಂಗಳೂರು"}, + {"input": "ಕರ್ನಾಟಕ.", "expected": "ಕರ್ನಾಟಕ"}, + + // 24. Korean (ko) - full + {"input": "안녕하세요!", "expected": "안녕하세요"}, + {"input": "서울", "expected": "서울"}, + {"input": "한국어.", "expected": "한국어"}, + {"input": "감사합니다?", "expected": "감사합니다"}, + + // 25. Lithuanian (lt) - beta + {"input": "Labas!", "expected": "labas"}, + {"input": "Vilnius", "expected": "vilnius"}, + {"input": "Lietuva.", "expected": "lietuva"}, + {"input": "ačiū", "expected": "aciu"}, + + // 26. Latvian (lv) - beta + {"input": "Sveiki!", "expected": "sveiki"}, + {"input": "Rīga", "expected": "riga"}, + {"input": "Latvija.", "expected": "latvija"}, + + // 27. Malay (ms) - beta + {"input": "Selamat pagi!", "expected": "selamat pagi"}, + {"input": "Kuala Lumpur", "expected": "kuala lumpur"}, + {"input": "Malaysia.", "expected": "malaysia"}, + + // 28. Mongolian (mn) - beta + {"input": "Сайн байна уу!", "expected": "сайн байна уу"}, + {"input": "Улаанбаатар", "expected": "улаанбаатар"}, + {"input": "Монгол.", "expected": "монгол"}, + + // 29. Marathi (mr) - beta + {"input": "नमस्कार!", "expected": "नमस्कार"}, + {"input": "मुंबई", "expected": "मुंबई"}, + {"input": "महाराष्ट्र.", "expected": "महाराष्ट्र"}, + + // 30. Dutch (nl) - beta + {"input": "Hallo!", "expected": "hallo"}, + {"input": "Amsterdam", "expected": "amsterdam"}, + {"input": "Nederland.", "expected": "nederland"}, + {"input": "dankjewel", "expected": "dankjewel"}, + + // 31. Punjabi (pa) - beta + {"input": "ਸਤਿ ਸ਼੍ਰੀ ਅਕਾਲ!", "expected": "ਸਤਿ ਸ਼੍ਰੀ ਅਕਾਲ"}, + {"input": "ਪੰਜਾਬ", "expected": "ਪੰਜਾਬ"}, + {"input": "ਅੰਮ੍ਰਿਤਸਰ.", "expected": "ਅੰਮ੍ਰਿਤਸਰ"}, + + // 32. Polish (pl) - beta + {"input": "Cześć!", "expected": "czesc"}, + {"input": "Warszawa", "expected": "warszawa"}, + {"input": "Polska.", "expected": "polska"}, + {"input": "dziękuję", "expected": "dziekuje"}, + + // 33. Portuguese (pt) - full + {"input": "Olá!", "expected": "ola"}, + {"input": "Obrigado", "expected": "obrigado"}, + {"input": "São Paulo.", "expected": "sao paulo"}, + {"input": "coração", "expected": "coracao"}, + {"input": "não?", "expected": "nao"}, + + // 34. Romanian (ro) - beta + {"input": "Salut!", "expected": "salut"}, + {"input": "București", "expected": "bucuresti"}, + {"input": "România.", "expected": "romania"}, + {"input": "mulțumesc", "expected": "multumesc"}, + + // 35. Russian (ru) - full + {"input": "Привет!", "expected": "привет"}, + {"input": "Москва", "expected": "москва"}, + {"input": "Россия.", "expected": "россия"}, + {"input": "спасибо?", "expected": "спасибо"}, + {"input": "магазин", "expected": "магазин"}, + {"input": "магазин.", "expected": "магазин"}, + + // 36. Slovak (sk) - beta + {"input": "Ahoj!", "expected": "ahoj"}, + {"input": "Bratislava", "expected": "bratislava"}, + {"input": "Slovensko.", "expected": "slovensko"}, + {"input": "ďakujem", "expected": "dakujem"}, + + // 37. Serbian (sr) - beta + {"input": "Здраво!", "expected": "здраво"}, + {"input": "Београд", "expected": "београд"}, + {"input": "Србија.", "expected": "србија"}, + + // 38. Ukrainian (uk) - beta + {"input": "Привіт!", "expected": "привіт"}, + {"input": "Київ", "expected": "київ"}, + {"input": "Україна.", "expected": "україна"}, + + // 39. Urdu (ur) - beta + {"input": "السلام علیکم!", "expected": "السلام علیکم"}, + {"input": "کراچی", "expected": "کراچی"}, + {"input": "پاکستان.", "expected": "پاکستان"}, + + // 40. Vietnamese (vi) - full + {"input": "Xin chào!", "expected": "xin chao"}, + {"input": "Hà Nội", "expected": "ha noi"}, + {"input": "Việt Nam.", "expected": "viet nam"}, + {"input": "cảm ơn?", "expected": "cam on"}, + + // 41. Cantonese (yue) - beta + {"input": "你好！", "expected": "你好"}, + {"input": "香港", "expected": "香港"}, + {"input": "廣東話.", "expected": "廣東話"}, + + // 42. Chinese Simplified (zh-CN) - full + {"input": "你好！", "expected": "你好"}, + {"input": "北京", "expected": "北京"}, + {"input": "中国.", "expected": "中国"}, + {"input": "谢谢?", "expected": "谢谢"}, + + // 43. Chinese Traditional (zh-TW) - full + {"input": "您好！", "expected": "您好"}, + {"input": "台北", "expected": "台北"}, + {"input": "台灣.", "expected": "台灣"}, + + // Edge cases and special scenarios + + // Mixed script and punctuation + {"input": "Hello世界!", "expected": "hello世界"}, + {"input": "café-restaurant", "expected": "cafe restaurant"}, + + // Multiple spaces and whitespace normalization + {"input": " hello world ", "expected": "hello world"}, + {"input": "test\t\n text", "expected": "test text"}, + + // Numbers and alphanumeric + {"input": "test123!", "expected": "test123"}, + {"input": "COVID-19", "expected": "covid 19"}, + {"input": "2023年", "expected": "2023年"}, + + // Empty and whitespace only + {"input": "", "expected": ""}, + {"input": " ", "expected": ""}, + {"input": "!!!", "expected": ""}, + + // Special punctuation combinations + {"input": "What?!?", "expected": "what"}, + {"input": "Well...", "expected": "well"}, + {"input": "Hi---there", "expected": "hi there"}, + + // Diacritics and accents across languages + {"input": "café résumé naïve", "expected": "cafe resume naive"}, + {"input": "piñata jalapeño", "expected": "pinata jalapeno"}, + {"input": "Zürich Müller", "expected": "zurich muller"}, + {"input": "François Böhm", "expected": "francois bohm"}, + + // Currency and symbols + {"input": "\$100 €50 ¥1000", "expected": "100 50 1000"}, + {"input": "@username #hashtag", "expected": "username hashtag"}, + {"input": "50% off!", "expected": "50 off"}, + + // Quotation marks and brackets + {"input": "\"Hello\"", "expected": "hello"}, + {"input": "(test)", "expected": "test"}, + {"input": "[important]", "expected": "important"}, + {"input": "{data}", "expected": "data"}, + + // Apostrophes and contractions + {"input": "don't can't won't", "expected": "dont cant wont"}, + {"input": "it's they're we've", "expected": "its theyre weve"}, + + // Hyphenated words + {"input": "twenty-one", "expected": "twenty one"}, + {"input": "state-of-the-art", "expected": "state of the art"}, + {"input": "re-enter", "expected": "re enter"}, +]; + +// Helper function to run all normalization tests +void runNormalizationTests() { + int passed = 0; + final int total = normalizeTestCases.length; + + for (int i = 0; i < normalizeTestCases.length; i++) { + final testCase = normalizeTestCases[i]; + final input = testCase['input']!; + final expected = testCase['expected']!; + final actual = normalizeString(input, 'en'); // Default to English for tests + + if (actual == expected) { + passed++; + Logs().i('✓ Test ${i + 1} PASSED: "$input" → "$actual"'); + } else { + Logs().i( + '✗ Test ${i + 1} FAILED: "$input" → "$actual" (expected: "$expected")', + ); + } + } + + Logs().i( + '\nTest Results: $passed/$total tests passed (${(passed / total * 100).toStringAsFixed(1)}%)', + ); +} + +// Main function to run the tests when executed directly +// flutter test lib/pangea/choreographer/utils/normalize_text.dart +void main() { + group('Normalize String Tests', () { + for (int i = 0; i < normalizeTestCases.length; i++) { + final testCase = normalizeTestCases[i]; + final input = testCase['input']!; + final expected = testCase['expected']!; + + test('Test ${i + 1}: "$input" should normalize to "$expected"', () { + final actual = + normalizeString(input, 'en'); // Default to English for tests + expect( + actual, + equals(expected), + reason: 'Input: "$input" → Got: "$actual" → Expected: "$expected"', + ); + }); + } + }); +} From 4153dbcd6b1c361c44e632e7a42eb3e67fbbaf4d Mon Sep 17 00:00:00 2001 From: ggurdin Date: Fri, 7 Nov 2025 09:13:46 -0500 Subject: [PATCH 04/11] remove redundant logic from text normalization function --- .../choreographer/utils/normalize_text.dart | 20 ++++++------------- 1 file changed, 6 insertions(+), 14 deletions(-) diff --git a/lib/pangea/choreographer/utils/normalize_text.dart b/lib/pangea/choreographer/utils/normalize_text.dart index 96eee9422..d510a71ca 100644 --- a/lib/pangea/choreographer/utils/normalize_text.dart +++ b/lib/pangea/choreographer/utils/normalize_text.dart @@ -7,10 +7,8 @@ import 'package:fluffychat/pangea/common/utils/error_handler.dart'; // We would like esta = está, hello! = Hello, etc. String normalizeString(String input, String languageCode) { try { - String normalized = input; - // Step 1: Convert to lowercase (works for all Unicode scripts) - normalized = normalized.toLowerCase(); + String normalized = input.toLowerCase(); // Step 2: Apply language-specific normalization rules normalized = _applyLanguageSpecificNormalization(normalized, languageCode); @@ -23,19 +21,13 @@ String normalizeString(String input, String languageCode) { // Step 4: Remove punctuation (including Unicode punctuation) // This removes ASCII and Unicode punctuation while preserving letters, numbers, and spaces - normalized = - normalized.replaceAll(RegExp(r'[\p{P}\p{S}]', unicode: true), ''); + normalized = normalized.replaceAll( + RegExp(r'[\p{P}\p{S}]', unicode: true), + '', + ); // Step 5: Normalize whitespace (collapse multiple spaces, trim) - normalized = normalized.replaceAll(RegExp(r'\s+'), ' ').trim(); - - // Step 6: Handle edge case where result becomes empty - if (normalized.isEmpty) { - // If normalization results in empty string, return empty string - return ''; - } - - return normalized; + return normalized.replaceAll(RegExp(r'\s+'), ' ').trim(); } catch (e, s) { ErrorHandler.logError( e: e, From 18ecb0008cc560779866f8149f389d7f1119c593 Mon Sep 17 00:00:00 2001 From: avashilling <165050625+avashilling@users.noreply.github.com> Date: Mon, 10 Nov 2025 12:17:28 -0500 Subject: [PATCH 05/11] add display name to activity summary --- .../activity_sessions/activity_room_extension.dart | 9 +++++++++ .../activity_summary/activity_summary_request_model.dart | 4 ++++ 2 files changed, 13 insertions(+) diff --git a/lib/pangea/activity_sessions/activity_room_extension.dart b/lib/pangea/activity_sessions/activity_room_extension.dart index df5bc9fef..ebbb97b71 100644 --- a/lib/pangea/activity_sessions/activity_room_extension.dart +++ b/lib/pangea/activity_sessions/activity_room_extension.dart @@ -238,9 +238,17 @@ extension ActivityRoomExtension on Room { continue; } + final sender = getParticipants().firstWhereOrNull( + (user) => user.id == event.senderId, + ); + final displayName = sender?.calcDisplayname() ?? + event.senderId.localpart ?? + event.senderId; + final activityMessage = event.messageType == MessageTypes.Text ? ActivitySummaryResultsMessage( userId: event.senderId, + displayName: displayName, sent: pangeaMessage.originalSent?.text ?? event.body, written: pangeaMessage.originalWrittenContent, time: event.originServerTs, @@ -253,6 +261,7 @@ extension ActivityRoomExtension on Room { ) : ActivitySummaryResultsMessage( userId: event.senderId, + displayName: displayName, sent: pangeaMessage.getSpeechToTextLocal()!.transcript.text.trim(), written: diff --git a/lib/pangea/activity_summary/activity_summary_request_model.dart b/lib/pangea/activity_summary/activity_summary_request_model.dart index 0334be896..2979aba06 100644 --- a/lib/pangea/activity_summary/activity_summary_request_model.dart +++ b/lib/pangea/activity_summary/activity_summary_request_model.dart @@ -7,6 +7,7 @@ import 'package:fluffychat/pangea/activity_summary/activity_summary_response_mod class ActivitySummaryResultsMessage { final String userId; + final String displayName; final String sent; final String? written; final List tool; @@ -14,6 +15,7 @@ class ActivitySummaryResultsMessage { ActivitySummaryResultsMessage({ required this.userId, + required this.displayName, required this.sent, this.written, required this.tool, @@ -23,6 +25,7 @@ class ActivitySummaryResultsMessage { factory ActivitySummaryResultsMessage.fromJson(Map json) { return ActivitySummaryResultsMessage( userId: json['user_id'] as String, + displayName: json['display_name'] as String, sent: json['sent'] as String, written: json['written'] as String?, tool: (json['tool'] as List).map((e) => e as String).toList(), @@ -33,6 +36,7 @@ class ActivitySummaryResultsMessage { Map toJson() { return { 'user_id': userId, + 'display_name': displayName, 'sent': sent, if (written != null) 'written': written, 'tool': tool, From 4fb906160c9fc580f519c8614732a7fe8461b9f8 Mon Sep 17 00:00:00 2001 From: Kelrap Date: Mon, 10 Nov 2025 15:19:29 -0500 Subject: [PATCH 06/11] Move chat view safe area to include background image --- lib/pages/chat/chat_view.dart | 71 ++++++++++++++++++----------------- 1 file changed, 36 insertions(+), 35 deletions(-) diff --git a/lib/pages/chat/chat_view.dart b/lib/pages/chat/chat_view.dart index 0e60f595f..a72f45890 100644 --- a/lib/pages/chat/chat_view.dart +++ b/lib/pages/chat/chat_view.dart @@ -329,32 +329,33 @@ class ChatView extends StatelessWidget { // onDragEntered: controller.onDragEntered, // onDragExited: controller.onDragExited, // child: Stack( - body: Stack( - // Pangea# - children: [ - if (accountConfig.wallpaperUrl != null) - Opacity( - opacity: accountConfig.wallpaperOpacity ?? 0.5, - child: ImageFiltered( - imageFilter: ui.ImageFilter.blur( - sigmaX: accountConfig.wallpaperBlur ?? 0.0, - sigmaY: accountConfig.wallpaperBlur ?? 0.0, - ), - child: MxcImage( - cacheKey: accountConfig.wallpaperUrl.toString(), - uri: accountConfig.wallpaperUrl, - fit: BoxFit.cover, - height: MediaQuery.of(context).size.height, - width: MediaQuery.of(context).size.width, - isThumbnail: false, - placeholder: (_) => Container(), + body: SafeArea( + child: Stack( + // Pangea# + children: [ + if (accountConfig.wallpaperUrl != null) + Opacity( + opacity: accountConfig.wallpaperOpacity ?? 0.5, + child: ImageFiltered( + imageFilter: ui.ImageFilter.blur( + sigmaX: accountConfig.wallpaperBlur ?? 0.0, + sigmaY: accountConfig.wallpaperBlur ?? 0.0, + ), + child: MxcImage( + cacheKey: accountConfig.wallpaperUrl.toString(), + uri: accountConfig.wallpaperUrl, + fit: BoxFit.cover, + height: MediaQuery.of(context).size.height, + width: MediaQuery.of(context).size.width, + isThumbnail: false, + placeholder: (_) => Container(), + ), ), ), - ), - SafeArea( // #Pangea + // SafeArea( // child: Column( - child: Stack( + Stack( children: [ Column( // Pangea# @@ -529,19 +530,19 @@ class ChatView extends StatelessWidget { // Pangea# ], ), - ), - // #Pangea - // if (controller.dragging) - // Container( - // color: theme.scaffoldBackgroundColor.withAlpha(230), - // alignment: Alignment.center, - // child: const Icon( - // Icons.upload_outlined, - // size: 100, - // ), - // ), - // Pangea# - ], + // #Pangea + // if (controller.dragging) + // Container( + // color: theme.scaffoldBackgroundColor.withAlpha(230), + // alignment: Alignment.center, + // child: const Icon( + // Icons.upload_outlined, + // size: 100, + // ), + // ), + // Pangea# + ], + ), ), ); }, From 5dfcb5f1b573e28f207e709513b93c94d6d8f1d3 Mon Sep 17 00:00:00 2001 From: ggurdin Date: Mon, 10 Nov 2025 16:11:24 -0500 Subject: [PATCH 07/11] fix: filter rooms without the join rule public from public courses page --- lib/pangea/login/pages/public_courses_page.dart | 3 ++- pubspec.yaml | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/lib/pangea/login/pages/public_courses_page.dart b/lib/pangea/login/pages/public_courses_page.dart index 3a63f8771..e3fba542e 100644 --- a/lib/pangea/login/pages/public_courses_page.dart +++ b/lib/pangea/login/pages/public_courses_page.dart @@ -68,7 +68,8 @@ class PublicCoursesPageState extends State { r.id == c.room.roomId && r.membership == Membership.join, ) && - coursePlans.containsKey(c.courseId), + coursePlans.containsKey(c.courseId) && + c.room.joinRule == 'public', ) .toList(); diff --git a/pubspec.yaml b/pubspec.yaml index 2e153701d..e64febf56 100644 --- a/pubspec.yaml +++ b/pubspec.yaml @@ -6,7 +6,7 @@ description: Learn a language while texting your friends. # Pangea# publish_to: none # On version bump also increase the build number for F-Droid -version: 4.1.15+4 +version: 4.1.15+5 environment: sdk: ">=3.0.0 <4.0.0" From 0ab91ccb6837b1014c40013aee57aeeef487e233 Mon Sep 17 00:00:00 2001 From: ggurdin Date: Tue, 11 Nov 2025 13:45:22 -0500 Subject: [PATCH 08/11] feat: display user's subscription end/refresh date --- lib/l10n/intl_en.arb | 6 +- lib/pages/chat_list/chat_list.dart | 17 +++-- lib/pangea/common/constants/local.key.dart | 1 + .../controllers/subscription_controller.dart | 20 +++--- .../models/base_subscription_info.dart | 4 ++ .../models/mobile_subscriptions.dart | 6 +- .../models/web_subscriptions.dart | 11 +++- .../pages/change_subscription.dart | 19 +++--- .../pages/settings_subscription.dart | 65 +++++++++--------- .../pages/settings_subscription_view.dart | 66 +++++++++++++++---- .../repo/subscription_management_repo.dart | 18 +++++ .../subscription/repo/subscription_repo.dart | 11 ---- 12 files changed, 155 insertions(+), 89 deletions(-) diff --git a/lib/l10n/intl_en.arb b/lib/l10n/intl_en.arb index c35a3149b..a1557af98 100644 --- a/lib/l10n/intl_en.arb +++ b/lib/l10n/intl_en.arb @@ -5321,5 +5321,9 @@ "inviteFriends": "Invite friends", "activityStatsButtonTooltip": "Activity info", "allow": "Allow", - "deny": "Deny" + "deny": "Deny", + "enabledRenewal": "Enable Subscription Renewal", + "subscriptionEndsOn": "Subscription Ends On", + "subscriptionRenewsOn": "Subscription Renews On", + "waitForSubscriptionChanges": "Changes to your subscription may take a moment to reflect in the app." } \ No newline at end of file diff --git a/lib/pages/chat_list/chat_list.dart b/lib/pages/chat_list/chat_list.dart index bd402ab4d..37bdbd7b9 100644 --- a/lib/pages/chat_list/chat_list.dart +++ b/lib/pages/chat_list/chat_list.dart @@ -525,7 +525,6 @@ class ChatListController extends State //#Pangea StreamSubscription? _invitedSpaceSubscription; - StreamSubscription? _subscriptionStatusStream; StreamSubscription? _roomCapacitySubscription; //Pangea# @@ -613,13 +612,8 @@ class ChatListController extends State } }); - _subscriptionStatusStream ??= MatrixState - .pangeaController.subscriptionController.subscriptionStream.stream - .listen((event) { - if (mounted) { - showSubscribedSnackbar(context); - } - }); + MatrixState.pangeaController.subscriptionController.subscriptionNotifier + .addListener(_onSubscribe); // listen for space child updates for any space that is not the active space // so that when the user navigates to the space that was updated, it will @@ -673,6 +667,10 @@ class ChatListController extends State } // #Pangea + void _onSubscribe() { + if (mounted) showSubscribedSnackbar(context); + } + Future _joinInvitedSpaces() async { final invitedSpaces = Matrix.of(context).client.rooms.where( (r) => r.isSpace && r.membership == Membership.invite, @@ -691,8 +689,9 @@ class ChatListController extends State _intentUriStreamSubscription?.cancel(); //#Pangea _invitedSpaceSubscription?.cancel(); - _subscriptionStatusStream?.cancel(); _roomCapacitySubscription?.cancel(); + MatrixState.pangeaController.subscriptionController.subscriptionNotifier + .removeListener(_onSubscribe); //Pangea# scrollController.removeListener(_onScroll); super.dispose(); diff --git a/lib/pangea/common/constants/local.key.dart b/lib/pangea/common/constants/local.key.dart index 8c5d84061..9d722ca6c 100644 --- a/lib/pangea/common/constants/local.key.dart +++ b/lib/pangea/common/constants/local.key.dart @@ -4,6 +4,7 @@ class PLocalKey { static const String beganWebPayment = "beganWebPayment"; static const String dismissedPaywall = 'dismissedPaywall'; static const String paywallBackoff = 'paywallBackoff'; + static const String clickedCancelSubscription = 'clickedCancelSubscription'; static const String messagesSinceUpdate = 'messagesSinceLastUpdate'; static const String completedActivities = 'completedActivities'; static const String justInputtedCode = 'justInputtedCode'; diff --git a/lib/pangea/subscription/controllers/subscription_controller.dart b/lib/pangea/subscription/controllers/subscription_controller.dart index cca7bf4e0..16c126992 100644 --- a/lib/pangea/subscription/controllers/subscription_controller.dart +++ b/lib/pangea/subscription/controllers/subscription_controller.dart @@ -12,7 +12,6 @@ import 'package:url_launcher/url_launcher_string.dart'; import 'package:fluffychat/config/app_config.dart'; import 'package:fluffychat/l10n/l10n.dart'; import 'package:fluffychat/pangea/common/config/environment.dart'; -import 'package:fluffychat/pangea/common/controllers/base_controller.dart'; import 'package:fluffychat/pangea/common/controllers/pangea_controller.dart'; import 'package:fluffychat/pangea/common/network/requests.dart'; import 'package:fluffychat/pangea/common/network/urls.dart'; @@ -35,13 +34,13 @@ enum SubscriptionStatus { shouldShowPaywall, } -class SubscriptionController extends BaseController { +class SubscriptionController with ChangeNotifier { late PangeaController _pangeaController; CurrentSubscriptionInfo? currentSubscriptionInfo; AvailableSubscriptionsInfo? availableSubscriptionInfo; - final StreamController subscriptionStream = StreamController.broadcast(); + final ValueNotifier subscriptionNotifier = ValueNotifier(false); SubscriptionController(PangeaController pangeaController) : super() { _pangeaController = pangeaController; @@ -119,22 +118,20 @@ class SubscriptionController extends BaseController { (CustomerInfo info) async { final bool? wasSubscribed = isSubscribed; await updateCustomerInfo(); - if (wasSubscribed != null && - !wasSubscribed && - (isSubscribed != null && isSubscribed!)) { - subscriptionStream.add(true); + if (wasSubscribed == false && isSubscribed == true) { + subscriptionNotifier.value = true; } }, ); } else { if (SubscriptionManagementRepo.getBeganWebPayment()) { await SubscriptionManagementRepo.removeBeganWebPayment(); - if (isSubscribed != null && isSubscribed!) { - subscriptionStream.add(true); + if (isSubscribed == true) { + subscriptionNotifier.value = true; } } } - setState(null); + notifyListeners(); } catch (e, s) { debugPrint("Failed to initialize subscription controller"); ErrorHandler.logError( @@ -197,7 +194,6 @@ class SubscriptionController extends BaseController { isPromo: isPromo, ); await SubscriptionManagementRepo.setBeganWebPayment(); - setState(null); launchUrlString( paymentLink, webOnlyWindowName: "_self", @@ -234,7 +230,7 @@ class SubscriptionController extends BaseController { Future updateCustomerInfo() async { await currentSubscriptionInfo?.setCurrentSubscription(); - setState(null); + notifyListeners(); } /// if the user is subscribed, returns subscribed diff --git a/lib/pangea/subscription/models/base_subscription_info.dart b/lib/pangea/subscription/models/base_subscription_info.dart index a9ae276a0..61f6cae0f 100644 --- a/lib/pangea/subscription/models/base_subscription_info.dart +++ b/lib/pangea/subscription/models/base_subscription_info.dart @@ -11,6 +11,7 @@ class CurrentSubscriptionInfo { final AvailableSubscriptionsInfo availableSubscriptionInfo; DateTime? expirationDate; + DateTime? unsubscribeDetectedAt; String? currentSubscriptionId; CurrentSubscriptionInfo({ @@ -59,6 +60,9 @@ class CurrentSubscriptionInfo { (currentSubscription?.appId == availableSubscriptionInfo.appIds?.currentAppId); + DateTime? get subscriptionEndDate => + unsubscribeDetectedAt == null ? null : expirationDate; + void resetSubscription() => currentSubscriptionId = null; Future setCurrentSubscription() async {} } diff --git a/lib/pangea/subscription/models/mobile_subscriptions.dart b/lib/pangea/subscription/models/mobile_subscriptions.dart index 8d45469ae..23a73de64 100644 --- a/lib/pangea/subscription/models/mobile_subscriptions.dart +++ b/lib/pangea/subscription/models/mobile_subscriptions.dart @@ -104,10 +104,10 @@ class MobileSubscriptionInfo extends CurrentSubscriptionInfo { expirationDate = activeEntitlement.expirationDate != null ? DateTime.parse(activeEntitlement.expirationDate!) : null; + unsubscribeDetectedAt = activeEntitlement.unsubscribeDetectedAt != null + ? DateTime.parse(activeEntitlement.unsubscribeDetectedAt!) + : null; - if (activeEntitlement.periodType == PeriodType.trial) { - // We dont use actual trials as it would require adding a CC on devices - } if (currentSubscriptionId != null && currentSubscription == null) { Sentry.addBreadcrumb( Breadcrumb(message: "mismatch of productIds and currentSubscriptionID"), diff --git a/lib/pangea/subscription/models/web_subscriptions.dart b/lib/pangea/subscription/models/web_subscriptions.dart index 6f25f3513..2ceb36d11 100644 --- a/lib/pangea/subscription/models/web_subscriptions.dart +++ b/lib/pangea/subscription/models/web_subscriptions.dart @@ -21,7 +21,16 @@ class WebSubscriptionInfo extends CurrentSubscriptionInfo { ); currentSubscriptionId = rcResponse.currentSubscriptionId; - expirationDate = rcResponse.expirationDate; + final currentSubscription = + rcResponse.allSubscriptions?[currentSubscriptionId]; + + if (currentSubscription != null) { + expirationDate = DateTime.tryParse(currentSubscription.expiresDate); + unsubscribeDetectedAt = + currentSubscription.unsubscribeDetectedAt != null + ? DateTime.parse(currentSubscription.unsubscribeDetectedAt!) + : null; + } } catch (err) { currentSubscriptionId = AppConfig.errorSubscriptionId; } diff --git a/lib/pangea/subscription/pages/change_subscription.dart b/lib/pangea/subscription/pages/change_subscription.dart index a124abf09..d7a291c95 100644 --- a/lib/pangea/subscription/pages/change_subscription.dart +++ b/lib/pangea/subscription/pages/change_subscription.dart @@ -171,21 +171,20 @@ class ChangeSubscription extends StatelessWidget { ElevatedButton( onPressed: () => controller .submitChange(subscription), - child: Row( - mainAxisAlignment: - MainAxisAlignment.center, - children: [ - controller.loading - ? const CircularProgressIndicator - .adaptive() - : Text( + child: controller.loading + ? const LinearProgressIndicator() + : Row( + mainAxisAlignment: + MainAxisAlignment.center, + children: [ + Text( subscription.isTrial ? L10n.of(context) .activateTrial : L10n.of(context).pay, ), - ], - ), + ], + ), ), ], ), diff --git a/lib/pangea/subscription/pages/settings_subscription.dart b/lib/pangea/subscription/pages/settings_subscription.dart index 8def4f50d..a1f54d8ca 100644 --- a/lib/pangea/subscription/pages/settings_subscription.dart +++ b/lib/pangea/subscription/pages/settings_subscription.dart @@ -2,16 +2,16 @@ import 'dart:async'; import 'package:flutter/material.dart'; -import 'package:go_router/go_router.dart'; import 'package:url_launcher/url_launcher_string.dart'; import 'package:fluffychat/config/app_config.dart'; import 'package:fluffychat/pangea/common/config/environment.dart'; +import 'package:fluffychat/pangea/common/utils/error_handler.dart'; import 'package:fluffychat/pangea/subscription/controllers/subscription_controller.dart'; import 'package:fluffychat/pangea/subscription/pages/settings_subscription_view.dart'; +import 'package:fluffychat/pangea/subscription/repo/subscription_management_repo.dart'; import 'package:fluffychat/pangea/subscription/utils/subscription_app_id.dart'; import 'package:fluffychat/pangea/subscription/widgets/subscription_snackbar.dart'; -import 'package:fluffychat/widgets/future_loading_dialog.dart'; import 'package:fluffychat/widgets/matrix.dart'; class SubscriptionManagement extends StatefulWidget { @@ -27,37 +27,25 @@ class SubscriptionManagementController extends State { MatrixState.pangeaController.subscriptionController; SubscriptionDetails? selectedSubscription; - StreamSubscription? _subscriptionStatusStream; bool loading = false; - late StreamSubscription _settingsSubscription; - @override void initState() { if (!subscriptionController.initCompleter.isCompleted) { subscriptionController.initialize().then((_) => setState(() {})); } - _settingsSubscription = subscriptionController.stateStream.listen((event) { - debugPrint("stateStream event in subscription settings"); - setState(() {}); - }); - - _subscriptionStatusStream ??= - subscriptionController.subscriptionStream.stream.listen((_) { - showSubscribedSnackbar(context); - context.go('/rooms'); - }); - + subscriptionController.addListener(_onSubscriptionUpdate); + subscriptionController.subscriptionNotifier.addListener(_onSubscribe); subscriptionController.updateCustomerInfo(); super.initState(); } @override void dispose() { + subscriptionController.subscriptionNotifier.removeListener(_onSubscribe); + subscriptionController.removeListener(_onSubscriptionUpdate); super.dispose(); - _settingsSubscription.cancel(); - _subscriptionStatusStream?.cancel(); } bool get subscriptionsAvailable => @@ -106,29 +94,46 @@ class SubscriptionManagementController extends State { .currentSubscriptionInfo!.currentPlatformMatchesPurchasePlatform; } + DateTime? get expirationDate => + subscriptionController.currentSubscriptionInfo?.expirationDate; + + DateTime? get subscriptionEndDate => + subscriptionController.currentSubscriptionInfo?.subscriptionEndDate; + + void _onSubscriptionUpdate() => setState(() {}); + void _onSubscribe() => showSubscribedSnackbar(context); + Future submitChange( SubscriptionDetails subscription, { bool isPromo = false, }) async { setState(() => loading = true); - await showFutureLoadingDialog( - context: context, - future: () async => subscriptionController.submitSubscriptionChange( + try { + await subscriptionController.submitSubscriptionChange( subscription, context, isPromo: isPromo, - ), - onError: (error, s) { - setState(() => loading = false); - return null; - }, - ); - - if (mounted && loading) { - setState(() => loading = false); + ); + } catch (e, s) { + ErrorHandler.logError( + e: e, + s: s, + data: { + "subscription_id": subscription.id, + "is_promo": isPromo, + }, + ); + } finally { + if (mounted) setState(() => loading = false); } } + Future onClickCancelSubscription() async { + await SubscriptionManagementRepo.setClickedCancelSubscription(); + await launchMangementUrl(ManagementOption.cancel); + if (mounted) setState(() {}); + } + Future launchMangementUrl(ManagementOption option) async { String managementUrl = Environment.stripeManagementUrl; final String? email = diff --git a/lib/pangea/subscription/pages/settings_subscription_view.dart b/lib/pangea/subscription/pages/settings_subscription_view.dart index a7cb9e412..c5b3bb506 100644 --- a/lib/pangea/subscription/pages/settings_subscription_view.dart +++ b/lib/pangea/subscription/pages/settings_subscription_view.dart @@ -7,6 +7,7 @@ import 'package:intl/intl.dart'; import 'package:fluffychat/l10n/l10n.dart'; import 'package:fluffychat/pangea/subscription/pages/change_subscription.dart'; import 'package:fluffychat/pangea/subscription/pages/settings_subscription.dart'; +import 'package:fluffychat/pangea/subscription/repo/subscription_management_repo.dart'; import 'package:fluffychat/widgets/layouts/max_width_body.dart'; class SettingsSubscriptionView extends StatelessWidget { @@ -25,12 +26,18 @@ class SettingsSubscriptionView extends StatelessWidget { Column( children: [ ListTile( - title: Text(L10n.of(context).cancelSubscription), - enabled: controller.showManagementOptions, - onTap: () => controller.launchMangementUrl( - ManagementOption.cancel, + title: Text( + controller.subscriptionEndDate == null + ? L10n.of(context).cancelSubscription + : L10n.of(context).enabledRenewal, + ), + enabled: controller.showManagementOptions, + onTap: controller.onClickCancelSubscription, + trailing: Icon( + controller.subscriptionEndDate == null + ? Icons.cancel_outlined + : Icons.refresh_outlined, ), - trailing: const Icon(Icons.cancel_outlined), ), const Divider(height: 1), ListTile( @@ -49,6 +56,42 @@ class SettingsSubscriptionView extends StatelessWidget { ), enabled: controller.showManagementOptions, ), + if (controller.expirationDate != null) ...[ + const Divider(height: 1), + ListTile( + title: Text( + controller.subscriptionEndDate != null + ? L10n.of(context).subscriptionEndsOn + : L10n.of(context).subscriptionRenewsOn, + ), + subtitle: Text( + DateFormat.yMMMMd().format( + controller.expirationDate!.toLocal(), + ), + ), + ), + if (SubscriptionManagementRepo.getClickedCancelSubscription()) + Padding( + padding: const EdgeInsets.all(16.0), + child: Row( + spacing: 8.0, + mainAxisSize: MainAxisSize.min, + children: [ + const Icon( + Icons.info_outline, + size: 20, + ), + Flexible( + child: Text( + L10n.of(context).waitForSubscriptionChanges, + textAlign: TextAlign.center, + style: const TextStyle(fontStyle: FontStyle.italic), + ), + ), + ], + ), + ), + ], ], ), ]; @@ -68,16 +111,15 @@ class SettingsSubscriptionView extends StatelessWidget { child: Column( children: [ if (isSubscribed == null) - const Center(child: CircularProgressIndicator.adaptive()), - if (isSubscribed != null && - isSubscribed && - !controller.showManagementOptions) + const Center(child: CircularProgressIndicator.adaptive()) + else if (isSubscribed && !controller.showManagementOptions) ManagementNotAvailableWarning( controller: controller, - ), - if (isSubscribed != null && !isSubscribed) + ) + else if (isSubscribed && controller.showManagementOptions) + ...managementButtons + else ChangeSubscription(controller: controller), - if (controller.showManagementOptions) ...managementButtons, ], ), ), diff --git a/lib/pangea/subscription/repo/subscription_management_repo.dart b/lib/pangea/subscription/repo/subscription_management_repo.dart index c25d572d3..f0b96a678 100644 --- a/lib/pangea/subscription/repo/subscription_management_repo.dart +++ b/lib/pangea/subscription/repo/subscription_management_repo.dart @@ -69,4 +69,22 @@ class SubscriptionManagementRepo { final int backoff = _getPaywallBackoff() + 1; await _cache.write(PLocalKey.paywallBackoff, backoff); } + + static Future setClickedCancelSubscription() async { + await _cache.write( + PLocalKey.clickedCancelSubscription, + DateTime.now().toIso8601String(), + ); + } + + static bool getClickedCancelSubscription() { + final entry = _cache.read(PLocalKey.clickedCancelSubscription); + if (entry == null) return false; + final val = DateTime.tryParse(entry); + return val != null && DateTime.now().difference(val).inSeconds < 60; + } + + static Future removeClickedCancelSubscription() async { + await _cache.remove(PLocalKey.clickedCancelSubscription); + } } diff --git a/lib/pangea/subscription/repo/subscription_repo.dart b/lib/pangea/subscription/repo/subscription_repo.dart index 24ecfed98..f844a20b5 100644 --- a/lib/pangea/subscription/repo/subscription_repo.dart +++ b/lib/pangea/subscription/repo/subscription_repo.dart @@ -151,7 +151,6 @@ class RCProductsResponseModel { class RCSubscriptionResponseModel { String? currentSubscriptionId; SubscriptionDetails? currentSubscription; - DateTime? expirationDate; List? allEntitlements; Map? allSubscriptions; @@ -159,7 +158,6 @@ class RCSubscriptionResponseModel { this.currentSubscriptionId, this.currentSubscription, this.allEntitlements, - this.expirationDate, this.allSubscriptions, }); @@ -188,14 +186,6 @@ class RCSubscriptionResponseModel { } final String currentSubscriptionId = activeEntitlements[0]; - - final Map currentSubscriptionMetadata = - json['subscriptions'][currentSubscriptionId]; - - final DateTime expirationDate = DateTime.parse( - currentSubscriptionMetadata['expires_date'], - ); - final SubscriptionDetails? currentSubscription = allProducts?.firstWhereOrNull( (SubscriptionDetails sub) => @@ -206,7 +196,6 @@ class RCSubscriptionResponseModel { return RCSubscriptionResponseModel( currentSubscription: currentSubscription, currentSubscriptionId: currentSubscriptionId, - expirationDate: expirationDate, allEntitlements: activeEntitlements, allSubscriptions: history, ); From b06356f8c0703b8626dd92914f97817b7ed5a004 Mon Sep 17 00:00:00 2001 From: ggurdin Date: Tue, 11 Nov 2025 14:01:59 -0500 Subject: [PATCH 09/11] client-side replacement --- .../activity_sessions/activity_room_extension.dart | 9 --------- .../activity_user_summaries_widget.dart | 6 +++++- .../activity_summary/activity_summary_request_model.dart | 4 ---- .../activity_summary_response_model.dart | 3 +++ 4 files changed, 8 insertions(+), 14 deletions(-) diff --git a/lib/pangea/activity_sessions/activity_room_extension.dart b/lib/pangea/activity_sessions/activity_room_extension.dart index ebbb97b71..df5bc9fef 100644 --- a/lib/pangea/activity_sessions/activity_room_extension.dart +++ b/lib/pangea/activity_sessions/activity_room_extension.dart @@ -238,17 +238,9 @@ extension ActivityRoomExtension on Room { continue; } - final sender = getParticipants().firstWhereOrNull( - (user) => user.id == event.senderId, - ); - final displayName = sender?.calcDisplayname() ?? - event.senderId.localpart ?? - event.senderId; - final activityMessage = event.messageType == MessageTypes.Text ? ActivitySummaryResultsMessage( userId: event.senderId, - displayName: displayName, sent: pangeaMessage.originalSent?.text ?? event.body, written: pangeaMessage.originalWrittenContent, time: event.originServerTs, @@ -261,7 +253,6 @@ extension ActivityRoomExtension on Room { ) : ActivitySummaryResultsMessage( userId: event.senderId, - displayName: displayName, sent: pangeaMessage.getSpeechToTextLocal()!.transcript.text.trim(), written: diff --git a/lib/pangea/activity_sessions/activity_user_summaries_widget.dart b/lib/pangea/activity_sessions/activity_user_summaries_widget.dart index 3a4e69884..959c404b7 100644 --- a/lib/pangea/activity_sessions/activity_user_summaries_widget.dart +++ b/lib/pangea/activity_sessions/activity_user_summaries_widget.dart @@ -159,7 +159,11 @@ class ButtonControlledCarouselView extends StatelessWidget { Flexible( child: SingleChildScrollView( child: Text( - p.feedback, + p.displayFeedback( + user?.calcDisplayname() ?? + p.participantId.localpart ?? + p.participantId, + ), style: const TextStyle(fontSize: 12.0), ), ), diff --git a/lib/pangea/activity_summary/activity_summary_request_model.dart b/lib/pangea/activity_summary/activity_summary_request_model.dart index 2979aba06..0334be896 100644 --- a/lib/pangea/activity_summary/activity_summary_request_model.dart +++ b/lib/pangea/activity_summary/activity_summary_request_model.dart @@ -7,7 +7,6 @@ import 'package:fluffychat/pangea/activity_summary/activity_summary_response_mod class ActivitySummaryResultsMessage { final String userId; - final String displayName; final String sent; final String? written; final List tool; @@ -15,7 +14,6 @@ class ActivitySummaryResultsMessage { ActivitySummaryResultsMessage({ required this.userId, - required this.displayName, required this.sent, this.written, required this.tool, @@ -25,7 +23,6 @@ class ActivitySummaryResultsMessage { factory ActivitySummaryResultsMessage.fromJson(Map json) { return ActivitySummaryResultsMessage( userId: json['user_id'] as String, - displayName: json['display_name'] as String, sent: json['sent'] as String, written: json['written'] as String?, tool: (json['tool'] as List).map((e) => e as String).toList(), @@ -36,7 +33,6 @@ class ActivitySummaryResultsMessage { Map toJson() { return { 'user_id': userId, - 'display_name': displayName, 'sent': sent, if (written != null) 'written': written, 'tool': tool, diff --git a/lib/pangea/activity_summary/activity_summary_response_model.dart b/lib/pangea/activity_summary/activity_summary_response_model.dart index 9954ad59a..497fa1846 100644 --- a/lib/pangea/activity_summary/activity_summary_response_model.dart +++ b/lib/pangea/activity_summary/activity_summary_response_model.dart @@ -29,6 +29,9 @@ class ParticipantSummaryModel { 'superlatives': superlatives, }; } + + String displayFeedback(String displayName) => + feedback.replaceAll(participantId, displayName); } class ActivitySummaryResponseModel { From b120f3ef847e76f53cc2a2e223250c0401029e7c Mon Sep 17 00:00:00 2001 From: ggurdin Date: Tue, 11 Nov 2025 14:15:17 -0500 Subject: [PATCH 10/11] fix: exclude 'not proper' noun type from construct analytics popups --- lib/pangea/analytics_misc/get_analytics_controller.dart | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/lib/pangea/analytics_misc/get_analytics_controller.dart b/lib/pangea/analytics_misc/get_analytics_controller.dart index 78cda3b37..d07b78a8b 100644 --- a/lib/pangea/analytics_misc/get_analytics_controller.dart +++ b/lib/pangea/analytics_misc/get_analytics_controller.dart @@ -243,7 +243,14 @@ class GetAnalyticsController extends BaseController { } void _onUnlockMorphLemmas(Set unlocked) { - setState({'unlocked_constructs': unlocked}); + final Set excluded = { + ConstructIdentifier( + category: "NounType", + lemma: "Not_proper", + type: ConstructTypeEnum.morph, + ), + }; + setState({'unlocked_constructs': unlocked.difference(excluded)}); } /// A local cache of eventIds and construct uses for messages sent since the last update. From 92dfa52e47686f54bee44d5b52005806e74b6880 Mon Sep 17 00:00:00 2001 From: ggurdin Date: Tue, 11 Nov 2025 14:24:39 -0500 Subject: [PATCH 11/11] use lemma instead of construct ID for comparison --- .../analytics_misc/get_analytics_controller.dart | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/lib/pangea/analytics_misc/get_analytics_controller.dart b/lib/pangea/analytics_misc/get_analytics_controller.dart index d07b78a8b..726826225 100644 --- a/lib/pangea/analytics_misc/get_analytics_controller.dart +++ b/lib/pangea/analytics_misc/get_analytics_controller.dart @@ -243,14 +243,14 @@ class GetAnalyticsController extends BaseController { } void _onUnlockMorphLemmas(Set unlocked) { - final Set excluded = { - ConstructIdentifier( - category: "NounType", - lemma: "Not_proper", - type: ConstructTypeEnum.morph, - ), + const excludedLemmas = {'not_proper'}; + + final filtered = { + for (final id in unlocked) + if (!excludedLemmas.contains(id.lemma.toLowerCase())) id, }; - setState({'unlocked_constructs': unlocked.difference(excluded)}); + + setState({'unlocked_constructs': filtered}); } /// A local cache of eventIds and construct uses for messages sent since the last update.