bake language detections into pangea message events, change relative imports to absolute imports (#1947)

Co-authored-by: ggurdin <ggurdin@gmail.com>

lib/pangea/choreographer/controllers/choreographer.dart

@@ -11,7 +11,10 @@ import 'package:fluffychat/pangea/choreographer/controllers/alternative_translat
 import 'package:fluffychat/pangea/choreographer/controllers/igc_controller.dart';
 import 'package:fluffychat/pangea/choreographer/enums/assistance_state_enum.dart';
 import 'package:fluffychat/pangea/choreographer/enums/edit_type.dart';
+import 'package:fluffychat/pangea/choreographer/models/choreo_record.dart';
 import 'package:fluffychat/pangea/choreographer/models/it_step.dart';
+import 'package:fluffychat/pangea/choreographer/models/pangea_match_model.dart';
+import 'package:fluffychat/pangea/choreographer/widgets/igc/pangea_text_controller.dart';
 import 'package:fluffychat/pangea/choreographer/widgets/igc/paywall_card.dart';
 import 'package:fluffychat/pangea/common/controllers/pangea_controller.dart';
 import 'package:fluffychat/pangea/common/utils/any_state_holder.dart';
@@ -21,14 +24,11 @@ import 'package:fluffychat/pangea/events/models/pangea_token_model.dart';
 import 'package:fluffychat/pangea/events/models/representation_content_model.dart';
 import 'package:fluffychat/pangea/events/models/tokens_event_content_model.dart';
 import 'package:fluffychat/pangea/learning_settings/constants/language_constants.dart';
+import 'package:fluffychat/pangea/learning_settings/models/language_model.dart';
 import 'package:fluffychat/pangea/spaces/models/space_model.dart';
 import 'package:fluffychat/pangea/subscription/controllers/subscription_controller.dart';
 import 'package:fluffychat/pangea/toolbar/controllers/tts_controller.dart';
-import '../../../widgets/matrix.dart';
-import '../../learning_settings/models/language_model.dart';
-import '../models/choreo_record.dart';
-import '../models/pangea_match_model.dart';
-import '../widgets/igc/pangea_text_controller.dart';
+import 'package:fluffychat/widgets/matrix.dart';
 import 'error_service.dart';
 import 'it_controller.dart';
 
@@ -188,7 +188,10 @@ class Choreographer {
     );
 
     final PangeaMessageTokens? tokensSent = igc.igcTextData?.tokens != null
-        ? PangeaMessageTokens(tokens: igc.igcTextData!.tokens)
+        ? PangeaMessageTokens(
+            tokens: igc.igcTextData!.tokens,
+            detections: igc.igcTextData!.detections.detections,
+          )
         : null;
 
     chatController.send(

lib/pangea/choreographer/models/igc_text_data_model.dart

@@ -11,12 +11,12 @@ import 'package:fluffychat/pangea/choreographer/models/pangea_match_model.dart';
 import 'package:fluffychat/pangea/choreographer/models/span_card_model.dart';
 import 'package:fluffychat/pangea/choreographer/models/span_data.dart';
 import 'package:fluffychat/pangea/choreographer/repo/language_detection_repo.dart';
+import 'package:fluffychat/pangea/common/constants/model_keys.dart';
 import 'package:fluffychat/pangea/common/utils/error_handler.dart';
 import 'package:fluffychat/pangea/events/event_wrappers/pangea_representation_event.dart';
 import 'package:fluffychat/pangea/events/models/pangea_token_model.dart';
 import 'package:fluffychat/pangea/events/models/representation_content_model.dart';
 import 'package:fluffychat/pangea/learning_settings/constants/language_constants.dart';
-import '../../common/constants/model_keys.dart';
 
 // import 'package:language_tool/language_tool.dart';
 
@@ -102,13 +102,22 @@ class IGCTextData {
       originalInput = matches.first.match.fullText;
     }
 
+    final defaultDetections = LanguageDetectionResponse(
+      detections: [
+        LanguageDetection(langCode: content.langCode, confidence: 1),
+      ],
+      fullText: content.text,
+    );
+
+    final LanguageDetectionResponse detections = event.detections != null
+        ? LanguageDetectionResponse.fromJson({
+            "detections": event.detections,
+            "full_text": content.text,
+          })
+        : defaultDetections;
+
     return IGCTextData(
-      detections: LanguageDetectionResponse(
-        detections: [
-          LanguageDetection(langCode: content.langCode, confidence: 1),
-        ],
-        fullText: content.text,
-      ),
+      detections: detections,
       originalInput: originalInput,
       fullTextCorrection: content.text,
       tokens: tokens,
@@ -143,11 +152,6 @@ class IGCTextData {
   String get detectedLanguage {
     return detections.detections.firstOrNull?.langCode ??
         LanguageKeys.unknownLanguage;
-    // if (!(enableIGC && enableIT) || matches.isNotEmpty) {
-    //   return detections.highestValidatedDetection().langCode;
-    // } else {
-    //   return detections.highestConfidenceDetection.langCode;
-    // }
   }
 
   // reconstruct fullText based on accepted match

lib/pangea/choreographer/repo/language_detection_repo.dart

@@ -6,7 +6,6 @@ import 'package:fluffychat/pangea/choreographer/models/language_detection_model.
 import 'package:fluffychat/pangea/common/config/environment.dart';
 import 'package:fluffychat/pangea/common/network/requests.dart';
 import 'package:fluffychat/pangea/common/network/urls.dart';
-import 'package:fluffychat/pangea/learning_settings/constants/language_constants.dart';
 
 class LanguageDetectionRepo {
   static Future<LanguageDetectionResponse> get(
@@ -78,22 +77,4 @@ class LanguageDetectionResponse {
       'full_text': fullText,
     };
   }
-
-  /// Return the highest confidence detection.
-  /// If there are no detections, the unknown language detection is returned.
-  LanguageDetection get highestConfidenceDetection {
-    detections.sort((a, b) => b.confidence.compareTo(a.confidence));
-    return detections.firstOrNull ?? unknownLanguageDetection;
-  }
-
-  static const double languageDetectionConfidenceThreshold = 0.95;
-
-  /// Returns the highest validated detection based on the confidence threshold.
-  /// If the highest confidence detection is below the threshold, the unknown language
-  /// detection is returned.
-  LanguageDetection highestValidatedDetection({double? threshold}) =>
-      highestConfidenceDetection.confidence >=
-              (threshold ?? languageDetectionConfidenceThreshold)
-          ? highestConfidenceDetection
-          : unknownLanguageDetection;
 }

lib/pangea/common/constants/model_keys.dart

@@ -53,6 +53,7 @@ class ModelKey {
   static const String fullText = "full_text";
   static const String fullTextLang = "full_text_lang";
   static const String tokens = "tokens";
+  static const String allDetections = "all_detections";
   static const String srcLang = "src_lang";
   static const String tgtLang = "tgt_lang";
   static const String word = "word";

lib/pangea/events/controllers/message_data_controller.dart

@@ -7,20 +7,20 @@ import 'package:flutter/material.dart';
 import 'package:http/http.dart';
 import 'package:matrix/matrix.dart';
 
+import 'package:fluffychat/pangea/choreographer/repo/full_text_translation_repo.dart';
 import 'package:fluffychat/pangea/common/config/environment.dart';
 import 'package:fluffychat/pangea/common/controllers/base_controller.dart';
 import 'package:fluffychat/pangea/common/controllers/pangea_controller.dart';
 import 'package:fluffychat/pangea/common/network/requests.dart';
 import 'package:fluffychat/pangea/common/network/urls.dart';
+import 'package:fluffychat/pangea/common/utils/error_handler.dart';
+import 'package:fluffychat/pangea/events/constants/pangea_event_types.dart';
 import 'package:fluffychat/pangea/events/event_wrappers/pangea_message_event.dart';
 import 'package:fluffychat/pangea/events/models/pangea_token_model.dart';
 import 'package:fluffychat/pangea/events/models/representation_content_model.dart';
 import 'package:fluffychat/pangea/events/models/tokens_event_content_model.dart';
 import 'package:fluffychat/pangea/events/repo/token_api_models.dart';
 import 'package:fluffychat/pangea/extensions/pangea_room_extension.dart';
-import '../../choreographer/repo/full_text_translation_repo.dart';
-import '../../common/utils/error_handler.dart';
-import '../constants/pangea_event_types.dart';
 
 // TODO - make this static and take it out of the _pangeaController
 // will need to pass accessToken to the requests
@@ -106,7 +106,10 @@ class MessageDataController extends BaseController {
     if (repEventId != null && room != null) {
       room
           .sendPangeaEvent(
-            content: PangeaMessageTokens(tokens: res.tokens).toJson(),
+            content: PangeaMessageTokens(
+              tokens: res.tokens,
+              detections: res.detections,
+            ).toJson(),
             parentEventId: repEventId,
             type: PangeaEventTypes.tokens,
           )
@@ -238,7 +241,10 @@ class MessageDataController extends BaseController {
 
     try {
       await room.sendPangeaEvent(
-        content: PangeaMessageTokens(tokens: res.tokens).toJson(),
+        content: PangeaMessageTokens(
+          tokens: res.tokens,
+          detections: res.detections,
+        ).toJson(),
         parentEventId: repEventId,
         type: PangeaEventTypes.tokens,
       );

lib/pangea/events/event_wrappers/pangea_representation_event.dart

@@ -10,16 +10,17 @@ import 'package:matrix/src/utils/markdown.dart';
 import 'package:sentry_flutter/sentry_flutter.dart';
 
 import 'package:fluffychat/pangea/choreographer/event_wrappers/pangea_choreo_event.dart';
+import 'package:fluffychat/pangea/choreographer/models/choreo_record.dart';
+import 'package:fluffychat/pangea/choreographer/models/language_detection_model.dart';
+import 'package:fluffychat/pangea/common/utils/error_handler.dart';
+import 'package:fluffychat/pangea/events/constants/pangea_event_types.dart';
 import 'package:fluffychat/pangea/events/extensions/pangea_event_extension.dart';
 import 'package:fluffychat/pangea/events/models/pangea_token_model.dart';
+import 'package:fluffychat/pangea/events/models/representation_content_model.dart';
 import 'package:fluffychat/pangea/events/models/tokens_event_content_model.dart';
 import 'package:fluffychat/pangea/events/repo/token_api_models.dart';
-import '../../../widgets/matrix.dart';
-import '../../choreographer/models/choreo_record.dart';
-import '../../common/utils/error_handler.dart';
-import '../../learning_settings/constants/language_constants.dart';
-import '../constants/pangea_event_types.dart';
-import '../models/representation_content_model.dart';
+import 'package:fluffychat/pangea/learning_settings/constants/language_constants.dart';
+import 'package:fluffychat/widgets/matrix.dart';
 
 class RepresentationEvent {
   Event? _event;
@@ -66,6 +67,8 @@ class RepresentationEvent {
   bool get botAuthored =>
       content.originalSent == false && content.originalWritten == false;
 
+  List<LanguageDetection>? get detections => _tokens?.detections;
+
   List<PangeaToken>? get tokens {
     if (_tokens != null) return _tokens!.tokens;
 

lib/pangea/events/models/tokens_event_content_model.dart

@@ -1,5 +1,6 @@
 import 'dart:convert';
 
+import 'package:fluffychat/pangea/choreographer/models/language_detection_model.dart';
 import 'package:fluffychat/pangea/events/models/pangea_token_model.dart';
 
 /// this class lives within a [PangeaTokensEvent]
@@ -8,9 +9,11 @@ import 'package:fluffychat/pangea/events/models/pangea_token_model.dart';
 /// representation
 class PangeaMessageTokens {
   List<PangeaToken> tokens;
+  List<LanguageDetection>? detections;
 
   PangeaMessageTokens({
     required this.tokens,
+    this.detections,
   });
 
   factory PangeaMessageTokens.fromJson(Map<String, dynamic> json) {
@@ -23,19 +26,32 @@ class PangeaMessageTokens {
         : something is String
             ? jsonDecode(json[_tokensKey])
             : null;
+
+    final Iterable? detectionsIterable = json[_detectionsKey] is Iterable
+        ? json[_detectionsKey]
+        : json[_detectionsKey] is String
+            ? jsonDecode(json[_detectionsKey])
+            : null;
     return PangeaMessageTokens(
       tokens: tokensIterable
           .map((e) => PangeaToken.fromJson(e))
           .toList()
           .cast<PangeaToken>(),
+      detections: detectionsIterable
+          ?.map((e) => LanguageDetection.fromJson(e))
+          .toList()
+          .cast<LanguageDetection>(),
     );
   }
 
   static const _tokensKey = "tkns";
+  static const _detectionsKey = "detections";
 
   Map<String, dynamic> toJson() {
     final data = <String, dynamic>{};
     data[_tokensKey] = jsonEncode(tokens.map((e) => e.toJson()).toList());
+    data[_detectionsKey] =
+        jsonEncode(detections?.map((e) => e.toJson()).toList());
     return data;
   }
 }

lib/pangea/events/repo/token_api_models.dart

@@ -1,5 +1,6 @@
+import 'package:fluffychat/pangea/choreographer/models/language_detection_model.dart';
 import 'package:fluffychat/pangea/common/constants/model_keys.dart';
-import '../models/pangea_token_model.dart';
+import 'package:fluffychat/pangea/events/models/pangea_token_model.dart';
 
 class TokensRequestModel {
   /// the text to be tokenized
@@ -53,8 +54,13 @@ class TokensRequestModel {
 class TokensResponseModel {
   List<PangeaToken> tokens;
   String lang;
+  List<LanguageDetection> detections;
 
-  TokensResponseModel({required this.tokens, required this.lang});
+  TokensResponseModel({
+    required this.tokens,
+    required this.lang,
+    required this.detections,
+  });
 
   factory TokensResponseModel.fromJson(
     Map<String, dynamic> json,
@@ -67,5 +73,11 @@ class TokensResponseModel {
             .toList()
             .cast<PangeaToken>(),
         lang: json[ModelKey.lang],
+        detections: (json[ModelKey.allDetections] as Iterable)
+            .map<LanguageDetection>(
+              (e) => LanguageDetection.fromJson(e as Map<String, dynamic>),
+            )
+            .toList()
+            .cast<LanguageDetection>(),
       );
 }

lib/pangea/toolbar/widgets/word_zoom/lemma_widget.dart

@@ -72,10 +72,15 @@ class LemmaWidgetState extends State<LemmaWidget> {
         editEventId: widget.pangeaMessageEvent.eventId,
         originalSent: widget.pangeaMessageEvent.originalSent?.content,
         originalWritten: widget.pangeaMessageEvent.originalWritten?.content,
-        tokensSent: PangeaMessageTokens(tokens: existingTokens),
+        tokensSent: PangeaMessageTokens(
+          tokens: existingTokens,
+          detections: widget.pangeaMessageEvent.originalSent!.detections,
+        ),
         tokensWritten: widget.pangeaMessageEvent.originalWritten?.tokens != null
             ? PangeaMessageTokens(
                 tokens: widget.pangeaMessageEvent.originalWritten!.tokens!,
+                detections:
+                    widget.pangeaMessageEvent.originalWritten?.detections,
               )
             : null,
         choreo: widget.pangeaMessageEvent.originalSent?.choreo,

lib/pangea/toolbar/widgets/word_zoom/morphs/morphological_center_widget.dart

@@ -114,9 +114,15 @@ class MorphologicalCenterWidgetState extends State<MorphologicalCenterWidget> {
         editEventId: pm.eventId,
         originalSent: pm.originalSent?.content,
         originalWritten: pm.originalWritten?.content,
-        tokensSent: PangeaMessageTokens(tokens: existingTokens),
+        tokensSent: PangeaMessageTokens(
+          tokens: existingTokens,
+          detections: pm.originalSent?.detections,
+        ),
         tokensWritten: pm.originalWritten?.tokens != null
-            ? PangeaMessageTokens(tokens: pm.originalWritten!.tokens!)
+            ? PangeaMessageTokens(
+                tokens: pm.originalWritten!.tokens!,
+                detections: pm.originalWritten?.detections,
+              )
             : null,
         choreo: pm.originalSent?.choreo,
         messageTag: ModelKey.messageTagMorphEdit,