chore: phonetic transcription changes for bot audio messages

2025-06-20 11:35:25 -04:00 · 2025-06-20 11:35:25 -04:00 · d8911e7271
commit d8911e7271
parent f578119352
7 changed files with 127 additions and 76 deletions
--- a/lib/pages/chat/chat.dart
+++ b/lib/pages/chat/chat.dart
@ -14,6 +14,7 @@ import 'package:device_info_plus/device_info_plus.dart';
 import 'package:emoji_picker_flutter/emoji_picker_flutter.dart';
 import 'package:go_router/go_router.dart';
 import 'package:image_picker/image_picker.dart';
+import 'package:just_audio/just_audio.dart';
 import 'package:matrix/matrix.dart';
 import 'package:scroll_to_index/scroll_to_index.dart';
 import 'package:sentry_flutter/sentry_flutter.dart';
@ -26,6 +27,7 @@ import 'package:fluffychat/config/themes.dart';
 import 'package:fluffychat/l10n/l10n.dart';
 import 'package:fluffychat/pages/chat/chat_view.dart';
 import 'package:fluffychat/pages/chat/event_info_dialog.dart';
+import 'package:fluffychat/pages/chat/events/audio_player.dart';
 import 'package:fluffychat/pages/chat/recording_dialog.dart';
 import 'package:fluffychat/pages/chat_details/chat_details.dart';
 import 'package:fluffychat/pangea/analytics_misc/construct_type_enum.dart';
@ -33,6 +35,7 @@ import 'package:fluffychat/pangea/analytics_misc/constructs_model.dart';
 import 'package:fluffychat/pangea/analytics_misc/gain_points_animation.dart';
 import 'package:fluffychat/pangea/analytics_misc/level_up.dart';
 import 'package:fluffychat/pangea/analytics_misc/put_analytics_controller.dart';
+import 'package:fluffychat/pangea/bot/utils/bot_name.dart';
 import 'package:fluffychat/pangea/chat/utils/unlocked_morphs_snackbar.dart';
 import 'package:fluffychat/pangea/chat/widgets/event_too_large_dialog.dart';
 import 'package:fluffychat/pangea/choreographer/controllers/choreographer.dart';
@ -46,6 +49,7 @@ import 'package:fluffychat/pangea/common/utils/error_handler.dart';
 import 'package:fluffychat/pangea/common/utils/firebase_analytics.dart';
 import 'package:fluffychat/pangea/common/utils/overlay.dart';
 import 'package:fluffychat/pangea/events/event_wrappers/pangea_message_event.dart';
+import 'package:fluffychat/pangea/events/extensions/pangea_event_extension.dart';
 import 'package:fluffychat/pangea/events/models/pangea_token_model.dart';
 import 'package:fluffychat/pangea/events/models/representation_content_model.dart';
 import 'package:fluffychat/pangea/events/models/tokens_event_content_model.dart';
@ -149,6 +153,7 @@ class ChatController extends State<ChatPageWithRoom>

  StreamSubscription? _levelSubscription;
  StreamSubscription? _analyticsSubscription;
+  StreamSubscription? _botAudioSubscription;
  // Pangea#
  Room get room => sendingClient.getRoomById(roomId) ?? widget.room;

@ -473,6 +478,41 @@ class ChatController extends State<ChatPageWithRoom>
        ignorePointer: true,
      );
    });
+
+    _botAudioSubscription = room.client.onSync.stream
+        .where(
+      (update) => update.rooms?.join?[roomId]?.timeline?.events != null,
+    )
+        .listen((update) async {
+      final timeline = update.rooms!.join![roomId]!.timeline!;
+      final botAudioEvent = timeline.events!.firstWhereOrNull(
+        (e) =>
+            e.senderId == BotName.byEnvironment &&
+            e.content.tryGet<String>('msgtype') == MessageTypes.Audio,
+      );
+      if (botAudioEvent == null) return;
+
+      final matrix = Matrix.of(context);
+      matrix.voiceMessageEventId.value = botAudioEvent.eventId;
+      matrix.audioPlayer?.dispose();
+      matrix.audioPlayer = AudioPlayer();
+
+      final event = Event.fromMatrixEvent(botAudioEvent, room);
+      final audioFile = await event.getPangeaAudioFile();
+      debugPrint(
+        "audiofile: ${audioFile?.mimeType} ${audioFile?.bytes.length}",
+      );
+      if (audioFile == null) return;
+
+      matrix.audioPlayer!.setAudioSource(
+        BytesAudioSource(
+          audioFile.bytes,
+          audioFile.mimeType,
+        ),
+      );
+
+      matrix.audioPlayer!.play();
+    });
    // Pangea#
    _tryLoadTimeline();
    if (kIsWeb) {
@ -719,6 +759,7 @@ class ChatController extends State<ChatPageWithRoom>
    stopMediaStream.close();
    _levelSubscription?.cancel();
    _analyticsSubscription?.cancel();
+    _botAudioSubscription?.cancel();
    _router.routeInformationProvider.removeListener(_onRouteChanged);
    //Pangea#
    super.dispose();
--- a/lib/pages/chat/events/audio_player.dart
+++ b/lib/pages/chat/events/audio_player.dart
@ -256,10 +256,10 @@ class AudioPlayerState extends State<AudioPlayerWidget> {
    _onAudioPositionChanged =
        matrix.audioPlayer!.positionStream.listen((state) {
      // Pass current timestamp to overlay, so it can highlight as necessary
-      if (widget.matrixFile != null) {
+      if (widget.matrixFile?.tokens != null) {
        widget.overlayController?.highlightCurrentText(
          state.inMilliseconds,
-          widget.matrixFile!.tokens,
+          widget.matrixFile!.tokens!,
        );
      }
    });
--- a/lib/pangea/events/extensions/pangea_event_extension.dart
+++ b/lib/pangea/events/extensions/pangea_event_extension.dart
@ -59,24 +59,22 @@ extension PangeaEvent on Event {
        content.tryGetMap<String, dynamic>(ModelKey.transcription);
    final audioContent =
        content.tryGetMap<String, dynamic>('org.matrix.msc1767.audio');
-    if (transcription == null || audioContent == null) {
-      ErrorHandler.logError(
-        e: "Called getPangeaAudioFile on an audio message without transcription or audio content",
-        data: {},
-      );
-      return null;
-    }

    final matrixFile = await downloadAndDecryptAttachment();
-    final duration = audioContent.tryGet<int>('duration');
-    final waveform = audioContent.tryGetList<int>('waveform');
+
+    final duration = audioContent?.tryGet<int>('duration') ??
+        content.tryGetMap<String, dynamic>('info')?.tryGet<int>('duration');
+
+    final waveform = audioContent?.tryGetList<int>('waveform') ??
+        content
+            .tryGetMap<String, dynamic>('org.matrix.msc1767.audio')
+            ?.tryGetList<int>('waveform');

    // old audio messages will not have tokens
-    final tokensContent = transcription.tryGetList(ModelKey.tokens);
-    if (tokensContent == null) return null;
+    final tokensContent = transcription?.tryGetList(ModelKey.tokens);

    final tokens = tokensContent
-        .map((e) => TTSToken.fromJson(e as Map<String, dynamic>))
+        ?.map((e) => TTSToken.fromJson(e as Map<String, dynamic>))
        .toList();

    return PangeaAudioFile(
--- a/lib/pangea/phonetic_transcription/phonetic_transcription_widget.dart
+++ b/lib/pangea/phonetic_transcription/phonetic_transcription_widget.dart
@ -15,10 +15,13 @@ import 'package:fluffychat/widgets/matrix.dart';
 class PhoneticTranscriptionWidget extends StatefulWidget {
  final String text;
  final LanguageModel textLanguage;
+
  final TextStyle? style;
  final double? iconSize;
  final Color? iconColor;

+  final bool enabled;
+
  const PhoneticTranscriptionWidget({
    super.key,
    required this.text,
@ -26,6 +29,7 @@ class PhoneticTranscriptionWidget extends StatefulWidget {
    this.style,
    this.iconSize,
    this.iconColor,
+    this.enabled = true,
  });

  @override
@ -114,69 +118,74 @@ class _PhoneticTranscriptionWidgetState

  @override
  Widget build(BuildContext context) {
-    return HoverBuilder(
-      builder: (context, hovering) {
-        return GestureDetector(
-          onTap: () => _handleAudioTap(context),
-          child: AnimatedContainer(
-            duration: const Duration(milliseconds: 150),
-            decoration: BoxDecoration(
-              color: hovering
-                  ? Colors.grey.withAlpha((0.2 * 255).round())
-                  : Colors.transparent,
-              borderRadius: BorderRadius.circular(6),
-            ),
-            padding: const EdgeInsets.symmetric(horizontal: 8, vertical: 4),
-            child: Row(
-              mainAxisSize: MainAxisSize.min,
-              children: [
-                if (_error != null)
-                  Row(
-                    spacing: 8.0,
-                    children: [
-                      Icon(
-                        Icons.error_outline,
+    return IgnorePointer(
+      ignoring: !widget.enabled,
+      child: HoverBuilder(
+        builder: (context, hovering) {
+          return GestureDetector(
+            onTap: () => _handleAudioTap(context),
+            child: AnimatedContainer(
+              duration: const Duration(milliseconds: 150),
+              decoration: BoxDecoration(
+                color: hovering
+                    ? Colors.grey.withAlpha((0.2 * 255).round())
+                    : Colors.transparent,
+                borderRadius: BorderRadius.circular(6),
+              ),
+              padding: const EdgeInsets.symmetric(horizontal: 8, vertical: 4),
+              child: Row(
+                mainAxisSize: MainAxisSize.min,
+                children: [
+                  if (_error != null)
+                    Row(
+                      spacing: 8.0,
+                      children: [
+                        Icon(
+                          Icons.error_outline,
+                          size: widget.iconSize ?? 24,
+                          color: Theme.of(context).colorScheme.error,
+                        ),
+                        Text(
+                          L10n.of(context).failedToFetchTranscription,
+                          style: widget.style,
+                        ),
+                      ],
+                    )
+                  else if (_isLoading || _transcription == null)
+                    const SizedBox(
+                      width: 16,
+                      height: 16,
+                      child: CircularProgressIndicator.adaptive(),
+                    )
+                  else
+                    Flexible(
+                      child: Text(
+                        "/$_transcription/",
+                        style: widget.style ??
+                            Theme.of(context).textTheme.bodyMedium,
+                      ),
+                    ),
+                  const SizedBox(width: 8),
+                  if (_transcription != null &&
+                      _error == null &&
+                      widget.enabled)
+                    Tooltip(
+                      message: _isPlaying
+                          ? L10n.of(context).stop
+                          : L10n.of(context).playAudio,
+                      child: Icon(
+                        _isPlaying ? Icons.pause_outlined : Icons.volume_up,
                        size: widget.iconSize ?? 24,
-                        color: Theme.of(context).colorScheme.error,
+                        color: widget.iconColor ??
+                            Theme.of(context).iconTheme.color,
                      ),
-                      Text(
-                        L10n.of(context).failedToFetchTranscription,
-                        style: widget.style,
-                      ),
-                    ],
-                  )
-                else if (_isLoading || _transcription == null)
-                  const SizedBox(
-                    width: 16,
-                    height: 16,
-                    child: CircularProgressIndicator.adaptive(),
-                  )
-                else
-                  Flexible(
-                    child: Text(
-                      "/$_transcription/",
-                      style: widget.style ??
-                          Theme.of(context).textTheme.bodyMedium,
                    ),
-                  ),
-                const SizedBox(width: 8),
-                if (_transcription != null && _error == null)
-                  Tooltip(
-                    message: _isPlaying
-                        ? L10n.of(context).stop
-                        : L10n.of(context).playAudio,
-                    child: Icon(
-                      _isPlaying ? Icons.pause_outlined : Icons.volume_up,
-                      size: widget.iconSize ?? 24,
-                      color:
-                          widget.iconColor ?? Theme.of(context).iconTheme.color,
-                    ),
-                  ),
-              ],
+                ],
+              ),
            ),
-          ),
-        );
-      },
+          );
+        },
+      ),
    );
  }
 }
--- a/lib/pangea/toolbar/widgets/message_audio_card.dart
+++ b/lib/pangea/toolbar/widgets/message_audio_card.dart
@ -104,7 +104,7 @@ class MessageAudioCardState extends State<MessageAudioCard> {

 class PangeaAudioFile extends MatrixAudioFile {
  List<int>? waveform;
-  List<TTSToken> tokens;
+  List<TTSToken>? tokens;

  PangeaAudioFile({
    required super.bytes,
--- a/lib/pangea/toolbar/widgets/overlay_message.dart
+++ b/lib/pangea/toolbar/widgets/overlay_message.dart
@ -8,6 +8,7 @@ import 'package:fluffychat/l10n/l10n.dart';
 import 'package:fluffychat/pages/chat/chat.dart';
 import 'package:fluffychat/pages/chat/events/message_content.dart';
 import 'package:fluffychat/pages/chat/events/reply_content.dart';
+import 'package:fluffychat/pangea/bot/utils/bot_name.dart';
 import 'package:fluffychat/pangea/events/event_wrappers/pangea_message_event.dart';
 import 'package:fluffychat/pangea/events/extensions/pangea_event_extension.dart';
 import 'package:fluffychat/pangea/learning_settings/models/language_model.dart';
@ -202,6 +203,8 @@ class OverlayMessage extends StatelessWidget {
                                    textColor,
                                  ),
                                  iconColor: textColor,
+                                  enabled:
+                                      event.senderId != BotName.byEnvironment,
                                ),
                            ],
                          ),
--- a/lib/pangea/toolbar/widgets/select_mode_buttons.dart
+++ b/lib/pangea/toolbar/widgets/select_mode_buttons.dart
@ -113,10 +113,10 @@ class SelectModeButtonsState extends State<SelectModeButtons> {
    });

    _onAudioPositionChanged ??= _audioPlayer?.positionStream.listen((state) {
-      if (_audioBytes != null) {
+      if (_audioBytes?.tokens != null) {
        widget.overlayController.highlightCurrentText(
          state.inMilliseconds,
-          _audioBytes!.tokens,
+          _audioBytes!.tokens!,
        );
      }
    });