ember/fluffychat
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

word focus turned off and tts on Choice click (#1118)

Browse source 
* word focus turned off and tts on Choice click

* play audio on word selection
This commit is contained in:
wcjord 2024-11-29 17:47:00 -05:00 • committed by GitHub
parent 25ab5e54bc
commit 78cb3afe0b
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
15 changed files with 198 additions and 94 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
lib/pages/chat/chat.dart
View file

@ -140,6 +140,7 @@ class ChatController extends State<ChatPageWithRoom>
Timer? typingTimeout;
bool currentlyTyping = false;
// #Pangea
// bool dragging = false;
// void onDragEntered(_) => setState(() => dragging = true);

6
lib/pangea/choreographer/controllers/choreographer.dart
View file

@ -17,6 +17,7 @@ import 'package:fluffychat/pangea/models/tokens_event_content_model.dart';
import 'package:fluffychat/pangea/utils/any_state_holder.dart';
import 'package:fluffychat/pangea/utils/error_handler.dart';
import 'package:fluffychat/pangea/utils/overlay.dart';
import 'package:fluffychat/pangea/widgets/chat/tts_controller.dart';
import 'package:fluffychat/pangea/widgets/igc/paywall_card.dart';
import 'package:flutter/foundation.dart';
import 'package:flutter/material.dart';
@ -40,6 +41,7 @@ class Choreographer {
late IgcController igc;
late AlternativeTranslator altTranslator;
late ErrorService errorService;
final tts = TtsController();
bool isFetching = false;
int _timesClicked = 0;
@ -66,6 +68,8 @@ class Choreographer {
.subscriptionController.trialActivationStream.stream
.listen((_) => _onChangeListener);
tts.setupTTS();
clear();
}
@ -454,6 +458,7 @@ class Choreographer {
choreoRecord = ChoreoRecord.newRecord;
itController.clear();
igc.clear();
//@ggurdin - why is this commented out?
// errorService.clear();
_resetDebounceTimer();
}
@ -477,6 +482,7 @@ class Choreographer {
dispose() {
_textController.dispose();
trialStream?.cancel();
tts.dispose();
}
LanguageModel? get l2Lang {

12
lib/pangea/choreographer/widgets/choice_array.dart
View file

@ -2,6 +2,7 @@ import 'dart:developer';
import 'dart:math';
import 'package:collection/collection.dart';
import 'package:fluffychat/pangea/widgets/chat/tts_controller.dart';
import 'package:flutter/foundation.dart';
import 'package:flutter/material.dart';
import 'package:flutter_gen/gen_l10n/l10n.dart';
@ -20,6 +21,10 @@ class ChoicesArray extends StatefulWidget {
final String originalSpan;
final String Function(int) uniqueKeyForLayerLink;
/// If null then should not be used
/// We don't want tts in the case of L1 options
final TtsController? tts;
/// Used to unqiuely identify the keys for choices, in cases where multiple
/// choices could have identical text, like in back-to-back practice activities
final String? id;
@ -35,6 +40,7 @@ class ChoicesArray extends StatefulWidget {
required this.originalSpan,
required this.uniqueKeyForLayerLink,
required this.selectedChoiceIndex,
required this.tts,
this.isActive = true,
this.onLongPress,
this.id,
@ -73,7 +79,11 @@ class ChoicesArrayState extends State<ChoicesArray> {
theme: theme,
onLongPress: widget.isActive ? widget.onLongPress : null,
onPressed: widget.isActive
? widget.onPressed
? (String value, int index) {
widget.onPressed(value, index);
// TODO - what to pass here as eventID?
widget.tts?.tryToSpeak(value, context, null);
}
: (String value, int index) {
debugger(when: kDebugMode);
},

1
lib/pangea/choreographer/widgets/it_bar.dart
View file

@ -418,6 +418,7 @@ class ITChoices extends StatelessWidget {
onLongPress: (value, index) => showCard(context, index),
uniqueKeyForLayerLink: (int index) => "itChoices$index",
selectedChoiceIndex: null,
tts: controller.choreographer.tts,
);
} catch (e) {
debugger(when: kDebugMode);

3
lib/pangea/choreographer/widgets/translation_finished_flow.dart
View file

@ -67,7 +67,7 @@ class AlternativeTranslations extends StatelessWidget {
@override
Widget build(BuildContext context) {
return ChoicesArray(
originalSpan: controller.choreographer.itController.sourceText ?? "dummy",
originalSpan: controller.sourceText ?? "dummy",
isLoading:
controller.choreographer.altTranslator.loadingAlternativeTranslations,
// choices: controller.choreographer.altTranslator.similarityResponse.scores
@ -82,6 +82,7 @@ class AlternativeTranslations extends StatelessWidget {
},
uniqueKeyForLayerLink: (int index) => "altTranslation$index",
selectedChoiceIndex: null,
tts: controller.choreographer.tts,
);
}
}

10
lib/pangea/enum/activity_type_enum.dart
View file

@ -26,6 +26,16 @@ extension ActivityTypeExtension on ActivityTypeEnum {
}
}
bool get includeTTSOnClick {
switch (this) {
case ActivityTypeEnum.wordMeaning:
return false;
case ActivityTypeEnum.wordFocusListening:
case ActivityTypeEnum.hiddenWordListening:
return true;
}
}
ActivityTypeEnum fromString(String value) {
final split = value.split('.').last;
switch (split) {

1
lib/pangea/models/pangea_token_model.dart
View file

@ -197,6 +197,7 @@ class PangeaToken {
case ActivityTypeEnum.wordMeaning:
return canBeDefined;
case ActivityTypeEnum.wordFocusListening:
return false;
case ActivityTypeEnum.hiddenWordListening:
return canBeHeard;
}

14
lib/pangea/models/practice_activities.dart/practice_activity_model.dart
View file

@ -4,6 +4,7 @@ import 'package:collection/collection.dart';
import 'package:fluffychat/pangea/enum/activity_display_instructions_enum.dart';
import 'package:fluffychat/pangea/enum/activity_type_enum.dart';
import 'package:fluffychat/pangea/enum/construct_type_enum.dart';
import 'package:fluffychat/pangea/models/pangea_token_model.dart';
import 'package:fluffychat/pangea/models/practice_activities.dart/multiple_choice_activity_model.dart';
import 'package:fluffychat/pangea/utils/error_handler.dart';
import 'package:flutter/foundation.dart';
@ -200,13 +201,20 @@ class PracticeActivityRequest {
}
class PracticeActivityModel {
// deprecated in favor of targetTokens
final List<ConstructIdentifier> tgtConstructs;
// being added after creation from request info
// TODO - replace tgtConstructs with targetTokens in server return
List<PangeaToken>? targetTokens;
final String langCode;
final ActivityTypeEnum activityType;
final ActivityContent content;
PracticeActivityModel({
required this.tgtConstructs,
required this.targetTokens,
required this.langCode,
required this.activityType,
required this.content,
@ -244,6 +252,11 @@ class PracticeActivityModel {
activityType:
ActivityTypeEnum.wordMeaning.fromString(json['activity_type']),
content: ActivityContent.fromJson(contentMap),
targetTokens: json['target_tokens'] is List
? (json['target_tokens'] as List)
.map((e) => PangeaToken.fromJson(e as Map<String, dynamic>))
.toList()
: null,
);
}
@ -256,6 +269,7 @@ class PracticeActivityModel {
'lang_code': langCode,
'activity_type': activityType.string,
'content': content.toJson(),
'target_tokens': targetTokens?.map((e) => e.toJson()).toList(),
};
}

10
lib/pangea/widgets/chat/message_audio_card.dart
View file

@ -59,11 +59,11 @@ class MessageAudioCardState extends State<MessageAudioCard> {
@override
void didUpdateWidget(covariant oldWidget) {
if (oldWidget.selection != widget.selection && widget.selection != null) {
debugPrint('selection changed');
setSectionStartAndEndFromSelection();
playSelectionAudio();
}
// if (oldWidget.selection != widget.selection && widget.selection != null) {
// debugPrint('selection changed');
// setSectionStartAndEndFromSelection();
// playSelectionAudio();
// }
super.didUpdateWidget(oldWidget);
}

15
lib/pangea/widgets/chat/message_selection_overlay.dart
View file

@ -18,7 +18,6 @@ import 'package:fluffychat/pangea/widgets/chat/message_toolbar_buttons.dart';
import 'package:fluffychat/pangea/widgets/chat/overlay_footer.dart';
import 'package:fluffychat/pangea/widgets/chat/overlay_header.dart';
import 'package:fluffychat/pangea/widgets/chat/overlay_message.dart';
import 'package:fluffychat/pangea/widgets/chat/tts_controller.dart';
import 'package:fluffychat/widgets/avatar.dart';
import 'package:fluffychat/widgets/matrix.dart';
import 'package:flutter/foundation.dart';
@ -67,7 +66,6 @@ class MessageOverlayController extends State<MessageSelectionOverlay>
PangeaMessageEvent? get pangeaMessageEvent => widget._pangeaMessageEvent;
final TtsController tts = TtsController();
bool _isPlayingAudio = false;
bool get showToolbarButtons =>
@ -139,8 +137,6 @@ class MessageOverlayController extends State<MessageSelectionOverlay>
);
},
).listen((_) => setState(() {}));
tts.setupTTS();
}
MessageAnalyticsEntry? get messageAnalyticsEntry =>
@ -297,6 +293,14 @@ class MessageOverlayController extends State<MessageSelectionOverlay>
}
}
if (_selectedSpan != null) {
widget.chatController.choreographer.tts.tryToSpeak(
token.text.content,
context,
pangeaMessageEvent!.eventId,
);
}
setState(() {});
}
@ -450,7 +454,7 @@ class MessageOverlayController extends State<MessageSelectionOverlay>
void dispose() {
_animationController.dispose();
_reactionSubscription?.cancel();
tts.dispose();
super.dispose();
}
@ -562,7 +566,6 @@ class MessageOverlayController extends State<MessageSelectionOverlay>
MessageToolbar(
pangeaMessageEvent: pangeaMessageEvent!,
overLayController: this,
ttsController: tts,
),
const SizedBox(height: 8),
SizedBox(

6
lib/pangea/widgets/chat/message_toolbar.dart
View file

@ -27,15 +27,16 @@ const double minCardHeight = 70;
class MessageToolbar extends StatelessWidget {
final PangeaMessageEvent pangeaMessageEvent;
final MessageOverlayController overLayController;
final TtsController ttsController;
const MessageToolbar({
super.key,
required this.pangeaMessageEvent,
required this.overLayController,
required this.ttsController,
});
TtsController get ttsController =>
overLayController.widget.chatController.choreographer.tts;
Widget toolbarContent(BuildContext context) {
final bool subscribed =
MatrixState.pangeaController.subscriptionController.isSubscribed;
@ -135,7 +136,6 @@ class MessageToolbar extends StatelessWidget {
return PracticeActivityCard(
pangeaMessageEvent: pangeaMessageEvent,
overlayController: overLayController,
ttsController: ttsController,
);
default:
debugger(when: kDebugMode);

7
lib/pangea/widgets/chat/tts_controller.dart
View file

@ -145,7 +145,8 @@ class TtsController {
Future<void> tryToSpeak(
String text,
BuildContext context,
String eventID,
// TODO - make non-nullable again
String? eventID,
) async {
if (_isLanguageFullySupported) {
await _speak(text);
@ -157,7 +158,9 @@ class TtsController {
'availableLangCodes': _availableLangCodes,
},
);
await _showMissingVoicePopup(context, eventID);
if (eventID != null) {
await _showMissingVoicePopup(context, eventID);
}
}
}

1
lib/pangea/widgets/igc/span_card.dart
View file

@ -280,6 +280,7 @@ class WordMatchContent extends StatelessWidget {
uniqueKeyForLayerLink: (int index) =>
"wordMatch$index",
selectedChoiceIndex: controller.selectedChoiceIndex,
tts: controller.widget.scm.choreographer.tts,
),
const SizedBox(height: 12),
PromptAndFeedback(controller: controller),

57
lib/pangea/widgets/practice_activity/multiple_choice_activity.dart
View file

@ -5,6 +5,7 @@ import 'package:fluffychat/config/app_config.dart';
import 'package:fluffychat/pangea/choreographer/widgets/choice_array.dart';
import 'package:fluffychat/pangea/controllers/put_analytics_controller.dart';
import 'package:fluffychat/pangea/enum/activity_type_enum.dart';
import 'package:fluffychat/pangea/models/pangea_token_model.dart';
import 'package:fluffychat/pangea/models/practice_activities.dart/practice_activity_model.dart';
import 'package:fluffychat/pangea/models/practice_activities.dart/practice_activity_record_model.dart';
import 'package:fluffychat/pangea/utils/error_handler.dart';
@ -21,7 +22,6 @@ import 'package:matrix/matrix.dart';
class MultipleChoiceActivity extends StatefulWidget {
final PracticeActivityCardState practiceCardController;
final PracticeActivityModel currentActivity;
final TtsController tts;
final Event event;
final VoidCallback? onError;
@ -29,7 +29,6 @@ class MultipleChoiceActivity extends StatefulWidget {
super.key,
required this.practiceCardController,
required this.currentActivity,
required this.tts,
required this.event,
this.onError,
});
@ -46,6 +45,8 @@ class MultipleChoiceActivityState extends State<MultipleChoiceActivity> {
@override
void initState() {
speakTargetTokens();
super.initState();
}
@ -55,18 +56,59 @@ class MultipleChoiceActivityState extends State<MultipleChoiceActivity> {
if (widget.practiceCardController.currentCompletionRecord?.responses
.isEmpty ??
false) {
speakTargetTokens();
setState(() => selectedChoiceIndex = null);
}
}
void speakTargetTokens() {
if (widget.practiceCardController.currentActivity?.targetTokens != null) {
widget.practiceCardController.tts.tryToSpeak(
PangeaToken.reconstructText(
widget.practiceCardController.currentActivity!.targetTokens!,
),
context,
null,
);
}
}
TtsController get tts => widget.practiceCardController.tts;
void updateChoice(String value, int index) {
final bool isCorrect =
widget.currentActivity.content.isCorrect(value, index);
// If the activity is not set to include TTS on click, and the choice is correct, speak the target tokens
// We have to check if tokens
if (!widget.currentActivity.activityType.includeTTSOnClick &&
isCorrect &&
mounted) {
// should be set by now but just in case we make a mistake
if (widget.practiceCardController.currentActivity?.targetTokens == null) {
debugger(when: kDebugMode);
ErrorHandler.logError(
e: "Missing target tokens in multiple choice activity",
data: {
"currentActivity": widget.practiceCardController.currentActivity,
},
);
} else {
tts.tryToSpeak(
PangeaToken.reconstructText(
widget.practiceCardController.currentActivity!.targetTokens!,
),
context,
null,
);
}
}
if (currentRecordModel?.hasTextResponse(value) ?? false) {
return;
}
final bool isCorrect =
widget.currentActivity.content.isCorrect(value, index);
currentRecordModel?.addResponse(
text: value,
score: isCorrect ? 1 : 0,
@ -136,7 +178,7 @@ class MultipleChoiceActivityState extends State<MultipleChoiceActivity> {
ActivityTypeEnum.wordFocusListening)
WordAudioButton(
text: practiceActivity.content.answer,
ttsController: widget.tts,
ttsController: tts,
eventID: widget.event.eventId,
),
if (practiceActivity.activityType ==
@ -146,7 +188,7 @@ class MultipleChoiceActivityState extends State<MultipleChoiceActivity> {
widget.practiceCardController.widget.pangeaMessageEvent,
overlayController:
widget.practiceCardController.widget.overlayController,
tts: widget.practiceCardController.widget.overlayController.tts,
tts: tts,
setIsPlayingAudio: widget.practiceCardController.widget
.overlayController.setIsPlayingAudio,
onError: widget.onError,
@ -170,6 +212,7 @@ class MultipleChoiceActivityState extends State<MultipleChoiceActivity> {
.toList(),
isActive: true,
id: currentRecordModel?.hashCode.toString(),
tts: practiceActivity.activityType.includeTTSOnClick ? tts : null,
),
],
);

148
lib/pangea/widgets/practice_activity/practice_activity_card.dart
View file

@ -30,13 +30,11 @@ import 'package:flutter/material.dart';
class PracticeActivityCard extends StatefulWidget {
final PangeaMessageEvent pangeaMessageEvent;
final MessageOverlayController overlayController;
final TtsController ttsController;
const PracticeActivityCard({
super.key,
required this.pangeaMessageEvent,
required this.overlayController,
required this.ttsController,
});
@override
@ -59,6 +57,9 @@ class PracticeActivityCardState extends State<PracticeActivityCard> {
Duration appropriateTimeForJoy = const Duration(milliseconds: 1500);
bool savoringTheJoy = false;
TtsController get tts =>
widget.overlayController.widget.chatController.choreographer.tts;
@override
void initState() {
super.initState();
@ -100,76 +101,86 @@ class PracticeActivityCardState extends State<PracticeActivityCard> {
Future<PracticeActivityModel?> _fetchActivity({
ActivityQualityFeedback? activityFeedback,
}) async {
// try {
debugPrint('Fetching activity');
_updateFetchingActivity(true);
try {
debugPrint('Fetching activity');
_updateFetchingActivity(true);
// target tokens can be empty if activities have been completed for each
// it's set on initialization and then removed when each activity is completed
if (!mounted ||
!pangeaController.languageController.languagesSet ||
widget.overlayController.messageAnalyticsEntry == null) {
// target tokens can be empty if activities have been completed for each
// it's set on initialization and then removed when each activity is completed
if (!mounted ||
!pangeaController.languageController.languagesSet ||
widget.overlayController.messageAnalyticsEntry == null) {
debugger(when: kDebugMode);
_updateFetchingActivity(false);
return null;
}
final nextActivitySpecs =
widget.overlayController.messageAnalyticsEntry?.nextActivity;
// the client is going to be choosing the next activity now
// if nothing is set then it must be done with practice
if (nextActivitySpecs == null) {
debugPrint("No next activity set, exiting practice flow");
_updateFetchingActivity(false);
return null;
}
// check if we already have an activity matching the specs
final existingActivity = practiceActivities.firstWhereOrNull(
(activity) =>
nextActivitySpecs.matchesActivity(activity.practiceActivity),
);
if (existingActivity != null) {
debugPrint('found existing activity');
_updateFetchingActivity(false);
existingActivity.practiceActivity.targetTokens =
nextActivitySpecs.tokens;
return existingActivity.practiceActivity;
}
debugPrint(
"client requesting ${nextActivitySpecs.activityType.string} for: ${nextActivitySpecs.tokens.map((t) => "word: ${t.text.content} xp: ${t.xp}").join(' ')}",
);
final PracticeActivityModelResponse? activityResponse =
await pangeaController.practiceGenerationController
.getPracticeActivity(
MessageActivityRequest(
userL1: pangeaController.languageController.userL1!.langCode,
userL2: pangeaController.languageController.userL2!.langCode,
messageText: widget.pangeaMessageEvent.messageDisplayText,
messageTokens: widget.overlayController.tokens!,
activityQualityFeedback: activityFeedback,
targetTokens: nextActivitySpecs.tokens,
targetType: nextActivitySpecs.activityType,
),
widget.pangeaMessageEvent,
);
currentActivityCompleter = activityResponse?.eventCompleter;
_updateFetchingActivity(false);
if (activityResponse == null || activityResponse.activity == null) {
debugPrint('No activity found');
return null;
}
activityResponse.activity!.targetTokens = nextActivitySpecs.tokens;
return activityResponse.activity;
} catch (e, s) {
debugger(when: kDebugMode);
_updateFetchingActivity(false);
ErrorHandler.logError(
e: e,
s: s,
m: 'Failed to get new activity',
data: {
'activity': currentActivity,
'record': currentCompletionRecord,
},
);
return null;
}
final nextActivitySpecs =
widget.overlayController.messageAnalyticsEntry?.nextActivity;
// the client is going to be choosing the next activity now
// if nothing is set then it must be done with practice
if (nextActivitySpecs == null) {
debugPrint("No next activity set, exiting practice flow");
_updateFetchingActivity(false);
return null;
}
// check if we already have an activity matching the specs
final existingActivity = practiceActivities.firstWhereOrNull(
(activity) =>
nextActivitySpecs.matchesActivity(activity.practiceActivity),
);
if (existingActivity != null) {
debugPrint('found existing activity');
_updateFetchingActivity(false);
return existingActivity.practiceActivity;
}
debugPrint(
"client requesting ${nextActivitySpecs.activityType.string} for: ${nextActivitySpecs.tokens.map((t) => "word: ${t.text.content} xp: ${t.xp}").join(' ')}",
);
final PracticeActivityModelResponse? activityResponse =
await pangeaController.practiceGenerationController.getPracticeActivity(
MessageActivityRequest(
userL1: pangeaController.languageController.userL1!.langCode,
userL2: pangeaController.languageController.userL2!.langCode,
messageText: widget.pangeaMessageEvent.messageDisplayText,
messageTokens: widget.overlayController.tokens!,
activityQualityFeedback: activityFeedback,
targetTokens: nextActivitySpecs.tokens,
targetType: nextActivitySpecs.activityType,
),
widget.pangeaMessageEvent,
);
currentActivityCompleter = activityResponse?.eventCompleter;
_updateFetchingActivity(false);
return activityResponse?.activity;
// } catch (e, s) {
// debugger(when: kDebugMode);
// ErrorHandler.logError(
// e: e,
// s: s,
// m: 'Failed to get new activity',
// data: {
// 'activity': currentActivity,
// 'record': currentCompletionRecord,
// },
// );
// return null;
// }
}
ConstructUseMetaData get metadata => ConstructUseMetaData(
@ -313,7 +324,6 @@ class PracticeActivityCardState extends State<PracticeActivityCard> {
return MultipleChoiceActivity(
practiceCardController: this,
currentActivity: currentActivity!,
tts: widget.ttsController,
event: widget.pangeaMessageEvent.event,
onError: _onError,
);