diff --git a/lib/pages/chat/chat.dart b/lib/pages/chat/chat.dart index 6ee5dc1dd..35639c8cf 100644 --- a/lib/pages/chat/chat.dart +++ b/lib/pages/chat/chat.dart @@ -14,6 +14,7 @@ import 'package:device_info_plus/device_info_plus.dart'; import 'package:emoji_picker_flutter/emoji_picker_flutter.dart'; import 'package:go_router/go_router.dart'; import 'package:image_picker/image_picker.dart'; +import 'package:just_audio/just_audio.dart'; import 'package:matrix/matrix.dart'; import 'package:scroll_to_index/scroll_to_index.dart'; import 'package:sentry_flutter/sentry_flutter.dart'; @@ -26,6 +27,7 @@ import 'package:fluffychat/config/themes.dart'; import 'package:fluffychat/l10n/l10n.dart'; import 'package:fluffychat/pages/chat/chat_view.dart'; import 'package:fluffychat/pages/chat/event_info_dialog.dart'; +import 'package:fluffychat/pages/chat/events/audio_player.dart'; import 'package:fluffychat/pages/chat/recording_dialog.dart'; import 'package:fluffychat/pages/chat_details/chat_details.dart'; import 'package:fluffychat/pangea/analytics_misc/construct_type_enum.dart'; @@ -33,6 +35,7 @@ import 'package:fluffychat/pangea/analytics_misc/constructs_model.dart'; import 'package:fluffychat/pangea/analytics_misc/gain_points_animation.dart'; import 'package:fluffychat/pangea/analytics_misc/level_up.dart'; import 'package:fluffychat/pangea/analytics_misc/put_analytics_controller.dart'; +import 'package:fluffychat/pangea/bot/utils/bot_name.dart'; import 'package:fluffychat/pangea/chat/utils/unlocked_morphs_snackbar.dart'; import 'package:fluffychat/pangea/chat/widgets/event_too_large_dialog.dart'; import 'package:fluffychat/pangea/choreographer/controllers/choreographer.dart'; @@ -46,6 +49,7 @@ import 'package:fluffychat/pangea/common/utils/error_handler.dart'; import 'package:fluffychat/pangea/common/utils/firebase_analytics.dart'; import 'package:fluffychat/pangea/common/utils/overlay.dart'; import 'package:fluffychat/pangea/events/event_wrappers/pangea_message_event.dart'; +import 'package:fluffychat/pangea/events/extensions/pangea_event_extension.dart'; import 'package:fluffychat/pangea/events/models/pangea_token_model.dart'; import 'package:fluffychat/pangea/events/models/representation_content_model.dart'; import 'package:fluffychat/pangea/events/models/tokens_event_content_model.dart'; @@ -149,6 +153,7 @@ class ChatController extends State StreamSubscription? _levelSubscription; StreamSubscription? _analyticsSubscription; + StreamSubscription? _botAudioSubscription; // Pangea# Room get room => sendingClient.getRoomById(roomId) ?? widget.room; @@ -473,6 +478,41 @@ class ChatController extends State ignorePointer: true, ); }); + + _botAudioSubscription = room.client.onSync.stream + .where( + (update) => update.rooms?.join?[roomId]?.timeline?.events != null, + ) + .listen((update) async { + final timeline = update.rooms!.join![roomId]!.timeline!; + final botAudioEvent = timeline.events!.firstWhereOrNull( + (e) => + e.senderId == BotName.byEnvironment && + e.content.tryGet('msgtype') == MessageTypes.Audio, + ); + if (botAudioEvent == null) return; + + final matrix = Matrix.of(context); + matrix.voiceMessageEventId.value = botAudioEvent.eventId; + matrix.audioPlayer?.dispose(); + matrix.audioPlayer = AudioPlayer(); + + final event = Event.fromMatrixEvent(botAudioEvent, room); + final audioFile = await event.getPangeaAudioFile(); + debugPrint( + "audiofile: ${audioFile?.mimeType} ${audioFile?.bytes.length}", + ); + if (audioFile == null) return; + + matrix.audioPlayer!.setAudioSource( + BytesAudioSource( + audioFile.bytes, + audioFile.mimeType, + ), + ); + + matrix.audioPlayer!.play(); + }); // Pangea# _tryLoadTimeline(); if (kIsWeb) { @@ -719,6 +759,7 @@ class ChatController extends State stopMediaStream.close(); _levelSubscription?.cancel(); _analyticsSubscription?.cancel(); + _botAudioSubscription?.cancel(); _router.routeInformationProvider.removeListener(_onRouteChanged); //Pangea# super.dispose(); diff --git a/lib/pages/chat/events/audio_player.dart b/lib/pages/chat/events/audio_player.dart index 40f9ba193..e7ab72b0c 100644 --- a/lib/pages/chat/events/audio_player.dart +++ b/lib/pages/chat/events/audio_player.dart @@ -256,10 +256,10 @@ class AudioPlayerState extends State { _onAudioPositionChanged = matrix.audioPlayer!.positionStream.listen((state) { // Pass current timestamp to overlay, so it can highlight as necessary - if (widget.matrixFile != null) { + if (widget.matrixFile?.tokens != null) { widget.overlayController?.highlightCurrentText( state.inMilliseconds, - widget.matrixFile!.tokens, + widget.matrixFile!.tokens!, ); } }); diff --git a/lib/pangea/events/extensions/pangea_event_extension.dart b/lib/pangea/events/extensions/pangea_event_extension.dart index d7cf70fe9..4a70baec1 100644 --- a/lib/pangea/events/extensions/pangea_event_extension.dart +++ b/lib/pangea/events/extensions/pangea_event_extension.dart @@ -59,24 +59,22 @@ extension PangeaEvent on Event { content.tryGetMap(ModelKey.transcription); final audioContent = content.tryGetMap('org.matrix.msc1767.audio'); - if (transcription == null || audioContent == null) { - ErrorHandler.logError( - e: "Called getPangeaAudioFile on an audio message without transcription or audio content", - data: {}, - ); - return null; - } final matrixFile = await downloadAndDecryptAttachment(); - final duration = audioContent.tryGet('duration'); - final waveform = audioContent.tryGetList('waveform'); + + final duration = audioContent?.tryGet('duration') ?? + content.tryGetMap('info')?.tryGet('duration'); + + final waveform = audioContent?.tryGetList('waveform') ?? + content + .tryGetMap('org.matrix.msc1767.audio') + ?.tryGetList('waveform'); // old audio messages will not have tokens - final tokensContent = transcription.tryGetList(ModelKey.tokens); - if (tokensContent == null) return null; + final tokensContent = transcription?.tryGetList(ModelKey.tokens); final tokens = tokensContent - .map((e) => TTSToken.fromJson(e as Map)) + ?.map((e) => TTSToken.fromJson(e as Map)) .toList(); return PangeaAudioFile( diff --git a/lib/pangea/phonetic_transcription/phonetic_transcription_widget.dart b/lib/pangea/phonetic_transcription/phonetic_transcription_widget.dart index d343c5855..8486a9991 100644 --- a/lib/pangea/phonetic_transcription/phonetic_transcription_widget.dart +++ b/lib/pangea/phonetic_transcription/phonetic_transcription_widget.dart @@ -15,10 +15,13 @@ import 'package:fluffychat/widgets/matrix.dart'; class PhoneticTranscriptionWidget extends StatefulWidget { final String text; final LanguageModel textLanguage; + final TextStyle? style; final double? iconSize; final Color? iconColor; + final bool enabled; + const PhoneticTranscriptionWidget({ super.key, required this.text, @@ -26,6 +29,7 @@ class PhoneticTranscriptionWidget extends StatefulWidget { this.style, this.iconSize, this.iconColor, + this.enabled = true, }); @override @@ -114,69 +118,74 @@ class _PhoneticTranscriptionWidgetState @override Widget build(BuildContext context) { - return HoverBuilder( - builder: (context, hovering) { - return GestureDetector( - onTap: () => _handleAudioTap(context), - child: AnimatedContainer( - duration: const Duration(milliseconds: 150), - decoration: BoxDecoration( - color: hovering - ? Colors.grey.withAlpha((0.2 * 255).round()) - : Colors.transparent, - borderRadius: BorderRadius.circular(6), - ), - padding: const EdgeInsets.symmetric(horizontal: 8, vertical: 4), - child: Row( - mainAxisSize: MainAxisSize.min, - children: [ - if (_error != null) - Row( - spacing: 8.0, - children: [ - Icon( - Icons.error_outline, + return IgnorePointer( + ignoring: !widget.enabled, + child: HoverBuilder( + builder: (context, hovering) { + return GestureDetector( + onTap: () => _handleAudioTap(context), + child: AnimatedContainer( + duration: const Duration(milliseconds: 150), + decoration: BoxDecoration( + color: hovering + ? Colors.grey.withAlpha((0.2 * 255).round()) + : Colors.transparent, + borderRadius: BorderRadius.circular(6), + ), + padding: const EdgeInsets.symmetric(horizontal: 8, vertical: 4), + child: Row( + mainAxisSize: MainAxisSize.min, + children: [ + if (_error != null) + Row( + spacing: 8.0, + children: [ + Icon( + Icons.error_outline, + size: widget.iconSize ?? 24, + color: Theme.of(context).colorScheme.error, + ), + Text( + L10n.of(context).failedToFetchTranscription, + style: widget.style, + ), + ], + ) + else if (_isLoading || _transcription == null) + const SizedBox( + width: 16, + height: 16, + child: CircularProgressIndicator.adaptive(), + ) + else + Flexible( + child: Text( + "/$_transcription/", + style: widget.style ?? + Theme.of(context).textTheme.bodyMedium, + ), + ), + const SizedBox(width: 8), + if (_transcription != null && + _error == null && + widget.enabled) + Tooltip( + message: _isPlaying + ? L10n.of(context).stop + : L10n.of(context).playAudio, + child: Icon( + _isPlaying ? Icons.pause_outlined : Icons.volume_up, size: widget.iconSize ?? 24, - color: Theme.of(context).colorScheme.error, + color: widget.iconColor ?? + Theme.of(context).iconTheme.color, ), - Text( - L10n.of(context).failedToFetchTranscription, - style: widget.style, - ), - ], - ) - else if (_isLoading || _transcription == null) - const SizedBox( - width: 16, - height: 16, - child: CircularProgressIndicator.adaptive(), - ) - else - Flexible( - child: Text( - "/$_transcription/", - style: widget.style ?? - Theme.of(context).textTheme.bodyMedium, ), - ), - const SizedBox(width: 8), - if (_transcription != null && _error == null) - Tooltip( - message: _isPlaying - ? L10n.of(context).stop - : L10n.of(context).playAudio, - child: Icon( - _isPlaying ? Icons.pause_outlined : Icons.volume_up, - size: widget.iconSize ?? 24, - color: - widget.iconColor ?? Theme.of(context).iconTheme.color, - ), - ), - ], + ], + ), ), - ), - ); - }, + ); + }, + ), ); } } diff --git a/lib/pangea/toolbar/widgets/message_audio_card.dart b/lib/pangea/toolbar/widgets/message_audio_card.dart index b4a4b26eb..2cc529422 100644 --- a/lib/pangea/toolbar/widgets/message_audio_card.dart +++ b/lib/pangea/toolbar/widgets/message_audio_card.dart @@ -104,7 +104,7 @@ class MessageAudioCardState extends State { class PangeaAudioFile extends MatrixAudioFile { List? waveform; - List tokens; + List? tokens; PangeaAudioFile({ required super.bytes, diff --git a/lib/pangea/toolbar/widgets/overlay_message.dart b/lib/pangea/toolbar/widgets/overlay_message.dart index 14dea0f1c..0ab11d738 100644 --- a/lib/pangea/toolbar/widgets/overlay_message.dart +++ b/lib/pangea/toolbar/widgets/overlay_message.dart @@ -8,6 +8,7 @@ import 'package:fluffychat/l10n/l10n.dart'; import 'package:fluffychat/pages/chat/chat.dart'; import 'package:fluffychat/pages/chat/events/message_content.dart'; import 'package:fluffychat/pages/chat/events/reply_content.dart'; +import 'package:fluffychat/pangea/bot/utils/bot_name.dart'; import 'package:fluffychat/pangea/events/event_wrappers/pangea_message_event.dart'; import 'package:fluffychat/pangea/events/extensions/pangea_event_extension.dart'; import 'package:fluffychat/pangea/learning_settings/models/language_model.dart'; @@ -202,6 +203,8 @@ class OverlayMessage extends StatelessWidget { textColor, ), iconColor: textColor, + enabled: + event.senderId != BotName.byEnvironment, ), ], ), diff --git a/lib/pangea/toolbar/widgets/select_mode_buttons.dart b/lib/pangea/toolbar/widgets/select_mode_buttons.dart index 7be70a778..b90611076 100644 --- a/lib/pangea/toolbar/widgets/select_mode_buttons.dart +++ b/lib/pangea/toolbar/widgets/select_mode_buttons.dart @@ -113,10 +113,10 @@ class SelectModeButtonsState extends State { }); _onAudioPositionChanged ??= _audioPlayer?.positionStream.listen((state) { - if (_audioBytes != null) { + if (_audioBytes?.tokens != null) { widget.overlayController.highlightCurrentText( state.inMilliseconds, - _audioBytes!.tokens, + _audioBytes!.tokens!, ); } });