From 711ae38f4a119fd5a69286f524cfaf019fd36f61 Mon Sep 17 00:00:00 2001 From: Kelrap <99418823+Kelrap@users.noreply.github.com> Date: Wed, 31 Dec 2025 09:26:06 -0500 Subject: [PATCH] Model key cleanup (#4983) * refactor: Group redundant ModelKey entries * Add python script to find and replace hardcoded ModelKey values * Edited Python script to not automatically use ModelKey for files not already using it * refactor: Ran script and accepted obvious changes * rename 'duration' model key --------- Co-authored-by: ggurdin --- .../activity_planner/activity_plan_model.dart | 18 +-- .../activity_plan_request.dart | 8 +- .../choreographer/igc/igc_request_model.dart | 4 +- .../choreographer/igc/igc_response_model.dart | 8 +- .../choreographer/igc/span_data_request.dart | 4 +- .../choreographer/it/it_request_model.dart | 8 +- lib/pangea/common/constants/model_keys.dart | 13 +- .../event_wrappers/pangea_message_event.dart | 6 +- .../extensions/pangea_event_extension.dart | 6 +- .../full_text_translation_request_model.dart | 2 +- lib/pangea/user/analytics_profile_model.dart | 12 +- lib/pangea/user/user_model.dart | 24 +-- scripts/find_hardcoded_keys.py | 142 ++++++++++++++++++ 13 files changed, 196 insertions(+), 59 deletions(-) create mode 100755 scripts/find_hardcoded_keys.py diff --git a/lib/pangea/activity_planner/activity_plan_model.dart b/lib/pangea/activity_planner/activity_plan_model.dart index 88478b1ff..7b72b877e 100644 --- a/lib/pangea/activity_planner/activity_plan_model.dart +++ b/lib/pangea/activity_planner/activity_plan_model.dart @@ -84,7 +84,7 @@ class ActivityPlanModel { instructions: json[ModelKey.activityPlanInstructions], req: req, title: json[ModelKey.activityPlanTitle], - description: json[ModelKey.activityPlanDescription] ?? + description: json[ModelKey.description] ?? json[ModelKey.activityPlanLearningObjective], learningObjective: json[ModelKey.activityPlanLearningObjective], vocab: List.from( @@ -93,11 +93,11 @@ class ActivityPlanModel { endAt: json[ModelKey.activityPlanEndAt] != null ? DateTime.parse(json[ModelKey.activityPlanEndAt]) : null, - duration: json[ModelKey.activityPlanDuration] != null + duration: json[ModelKey.duration] != null ? Duration( - days: json[ModelKey.activityPlanDuration]['days'] ?? 0, - hours: json[ModelKey.activityPlanDuration]['hours'] ?? 0, - minutes: json[ModelKey.activityPlanDuration]['minutes'] ?? 0, + days: json[ModelKey.duration]['days'] ?? 0, + hours: json[ModelKey.duration]['hours'] ?? 0, + minutes: json[ModelKey.duration]['minutes'] ?? 0, ) : null, roles: roles, @@ -113,11 +113,11 @@ class ActivityPlanModel { ModelKey.activityPlanInstructions: instructions, ModelKey.activityPlanRequest: req.toJson(), ModelKey.activityPlanTitle: title, - ModelKey.activityPlanDescription: description, + ModelKey.description: description, ModelKey.activityPlanLearningObjective: learningObjective, ModelKey.activityPlanVocab: vocab.map((vocab) => vocab.toJson()).toList(), ModelKey.activityPlanEndAt: endAt?.toIso8601String(), - ModelKey.activityPlanDuration: { + ModelKey.duration: { 'days': duration?.inDays ?? 0, 'hours': duration?.inHours.remainder(24) ?? 0, 'minutes': duration?.inMinutes.remainder(60) ?? 0, @@ -181,7 +181,7 @@ class Vocab { factory Vocab.fromJson(Map json) { return Vocab( - lemma: json['lemma'], + lemma: json[ModelKey.lemma], pos: json['pos'], ); } @@ -203,7 +203,7 @@ class Vocab { Map toJson() { return { - 'lemma': lemma, + ModelKey.lemma: lemma, 'pos': pos, }; } diff --git a/lib/pangea/activity_planner/activity_plan_request.dart b/lib/pangea/activity_planner/activity_plan_request.dart index 37c62beef..3e5672817 100644 --- a/lib/pangea/activity_planner/activity_plan_request.dart +++ b/lib/pangea/activity_planner/activity_plan_request.dart @@ -30,12 +30,12 @@ class ActivityPlanRequest { Map toJson() { return { ModelKey.activityRequestTopic: topic, - ModelKey.activityRequestMode: mode, + ModelKey.mode: mode, ModelKey.activityRequestObjective: objective, ModelKey.activityRequestMedia: media.string, ModelKey.activityRequestCefrLevel: cefrLevel.string, ModelKey.activityRequestLanguageOfInstructions: languageOfInstructions, - ModelKey.activityRequestTargetLanguage: targetLanguage, + ModelKey.targetLanguage: targetLanguage, ModelKey.activityRequestCount: count, ModelKey.activityRequestNumberOfParticipants: numberOfParticipants, ModelKey.activityPlanLocation: location, @@ -45,7 +45,7 @@ class ActivityPlanRequest { factory ActivityPlanRequest.fromJson(Map json) => ActivityPlanRequest( topic: json[ModelKey.activityRequestTopic], - mode: json[ModelKey.activityRequestMode], + mode: json[ModelKey.mode], objective: json[ModelKey.activityRequestObjective], media: MediaEnum.nan.fromString(json[ModelKey.activityRequestMedia]), cefrLevel: json[ModelKey.activityRequestCefrLevel] != null @@ -55,7 +55,7 @@ class ActivityPlanRequest { : LanguageLevelTypeEnum.a1, languageOfInstructions: json[ModelKey.activityRequestLanguageOfInstructions], - targetLanguage: json[ModelKey.activityRequestTargetLanguage], + targetLanguage: json[ModelKey.targetLanguage], count: json[ModelKey.activityRequestCount], numberOfParticipants: json[ModelKey.activityRequestNumberOfParticipants], diff --git a/lib/pangea/choreographer/igc/igc_request_model.dart b/lib/pangea/choreographer/igc/igc_request_model.dart index dc14b82fe..cbe6a41eb 100644 --- a/lib/pangea/choreographer/igc/igc_request_model.dart +++ b/lib/pangea/choreographer/igc/igc_request_model.dart @@ -25,8 +25,8 @@ class IGCRequestModel { ModelKey.fullText: fullText, ModelKey.userL1: userL1, ModelKey.userL2: userL2, - "enable_it": enableIT, - "enable_igc": enableIGC, + ModelKey.enableIT: enableIT, + ModelKey.enableIGC: enableIGC, ModelKey.userId: userId, ModelKey.prevMessages: jsonEncode(prevMessages.map((x) => x.toJson()).toList()), diff --git a/lib/pangea/choreographer/igc/igc_response_model.dart b/lib/pangea/choreographer/igc/igc_response_model.dart index fc1bd1c6f..aed6b4bcd 100644 --- a/lib/pangea/choreographer/igc/igc_response_model.dart +++ b/lib/pangea/choreographer/igc/igc_response_model.dart @@ -36,8 +36,8 @@ class IGCResponseModel { fullTextCorrection: json["full_text_correction"], userL1: json[ModelKey.userL1], userL2: json[ModelKey.userL2], - enableIT: json["enable_it"], - enableIGC: json["enable_igc"], + enableIT: json[ModelKey.enableIT], + enableIGC: json[ModelKey.enableIGC], ); } @@ -47,7 +47,7 @@ class IGCResponseModel { "matches": matches.map((e) => e.toJson()).toList(), ModelKey.userL1: userL1, ModelKey.userL2: userL2, - "enable_it": enableIT, - "enable_igc": enableIGC, + ModelKey.enableIT: enableIT, + ModelKey.enableIGC: enableIGC, }; } diff --git a/lib/pangea/choreographer/igc/span_data_request.dart b/lib/pangea/choreographer/igc/span_data_request.dart index 495eb6ac8..50fd6441b 100644 --- a/lib/pangea/choreographer/igc/span_data_request.dart +++ b/lib/pangea/choreographer/igc/span_data_request.dart @@ -19,8 +19,8 @@ class SpanDetailsRequest { Map toJson() => { ModelKey.userL1: userL1, ModelKey.userL2: userL2, - "enable_it": enableIT, - "enable_igc": enableIGC, + ModelKey.enableIT: enableIT, + ModelKey.enableIGC: enableIGC, 'span': span.toJson(), }; diff --git a/lib/pangea/choreographer/it/it_request_model.dart b/lib/pangea/choreographer/it/it_request_model.dart index 2420947de..b5baf2604 100644 --- a/lib/pangea/choreographer/it/it_request_model.dart +++ b/lib/pangea/choreographer/it/it_request_model.dart @@ -22,11 +22,11 @@ class ITRequestModel { }); factory ITRequestModel.fromJson(json) => ITRequestModel( - text: json['text'], + text: json[ModelKey.text], customInput: json['custom_input'], sourceLangCode: json[ModelKey.srcLang], targetLangCode: json[ModelKey.tgtLang], - goldTranslation: json['gold_translation'], + goldTranslation: json[ModelKey.goldTranslation], goldContinuances: json['gold_continuances'] != null ? (json['gold_continuances']) .map((e) => ContinuanceModel.fromJson(e)) @@ -35,11 +35,11 @@ class ITRequestModel { ); Map toJson() => { - 'text': text, + ModelKey.text: text, 'custom_input': customInput, ModelKey.srcLang: sourceLangCode, ModelKey.tgtLang: targetLangCode, - 'gold_translation': goldTranslation, + ModelKey.goldTranslation: goldTranslation, 'gold_continuances': goldContinuances != null ? List.from(goldContinuances!.map((e) => e.toJson())) : null, diff --git a/lib/pangea/common/constants/model_keys.dart b/lib/pangea/common/constants/model_keys.dart index 090777288..af68b1625 100644 --- a/lib/pangea/common/constants/model_keys.dart +++ b/lib/pangea/common/constants/model_keys.dart @@ -7,14 +7,10 @@ class ModelKey { static const String userCreatedAt = 'created_at'; static const String userPangeaUserId = 'pangea_user_id'; static const String userDateOfBirth = 'date_of_birth'; - static const String userTargetLanguage = 'target_language'; - static const String userSourceLanguage = 'source_language'; static const String userSpeaks = 'speaks'; static const String userCountry = 'country'; static const String hasJoinedHelpSpace = 'has_joined_help_space'; static const String userInterests = 'interests'; - static const String l2LanguageKey = 'target_language'; - static const String l1LanguageKey = 'source_language'; static const String publicProfile = 'public_profile'; static const String userId = 'user_id'; static const String toolSettings = 'tool_settings'; @@ -26,10 +22,8 @@ class ModelKey { static const String itAutoPlay = 'autoPlayIT'; static const String clientClassCity = "city"; - static const String clientClassCountry = "country"; static const String clientClassDominantLanguage = "dominantLanguage"; static const String clientClassTargetLanguage = "targetLanguage"; - static const String clientClassDescription = "description"; static const String clientLanguageLevel = "languageLevel"; static const String clientSchool = "schoolName"; @@ -136,6 +130,7 @@ class ModelKey { "text_adventure_game_master_instructions"; static const String targetLanguage = "target_language"; + static const String sourceLanguage = "source_language"; static const String targetVoice = "target_voice"; static const String prevEventId = "prev_event_id"; @@ -160,7 +155,8 @@ class ModelKey { // activity plan static const String activityPlanRequest = "req"; static const String activityPlanTitle = "title"; - static const String activityPlanDescription = "description"; + static const String description = "description"; + static const String duration = "duration"; static const String activityPlanLocation = "location"; static const String activityPlanLearningObjective = "learning_objective"; static const String activityPlanInstructions = "instructions"; @@ -168,17 +164,14 @@ class ModelKey { static const String activityPlanImageURL = "image_url"; static const String activityId = "activity_id"; static const String activityPlanEndAt = "end_at"; - static const String activityPlanDuration = "duration"; static const String activityPlanTopicId = "topic_id"; static const String activityRequestTopic = "topic"; - static const String activityRequestMode = "mode"; static const String activityRequestObjective = "objective"; static const String activityRequestMedia = "media"; static const String activityRequestCefrLevel = "activity_cefr_level"; static const String activityRequestLanguageOfInstructions = "language_of_instructions"; - static const String activityRequestTargetLanguage = "target_language"; static const String activityRequestCount = "count"; static const String activityRequestNumberOfParticipants = "number_of_participants"; diff --git a/lib/pangea/events/event_wrappers/pangea_message_event.dart b/lib/pangea/events/event_wrappers/pangea_message_event.dart index 5c572f369..b9839aa17 100644 --- a/lib/pangea/events/event_wrappers/pangea_message_event.dart +++ b/lib/pangea/events/event_wrappers/pangea_message_event.dart @@ -293,7 +293,7 @@ class PangeaMessageEvent { Event? getTextToSpeechLocal(String langCode, String text) { for (final audio in allAudio) { final dataMap = audio.content.tryGetMap(ModelKey.transcription); - if (dataMap == null || !dataMap.containsKey('tokens')) continue; + if (dataMap == null || !dataMap.containsKey(ModelKey.tokens)) continue; try { final PangeaAudioEventData audioData = PangeaAudioEventData.fromJson( @@ -413,11 +413,11 @@ class PangeaMessageEvent { extraContent: { 'info': { ...file.info, - 'duration': response.durationMillis, + ModelKey.duration: response.durationMillis, }, 'org.matrix.msc3245.voice': {}, 'org.matrix.msc1767.audio': { - 'duration': response.durationMillis, + ModelKey.duration: response.durationMillis, 'waveform': response.waveform, }, ModelKey.transcription: response diff --git a/lib/pangea/events/extensions/pangea_event_extension.dart b/lib/pangea/events/extensions/pangea_event_extension.dart index 4609a3ef0..6c71c7d22 100644 --- a/lib/pangea/events/extensions/pangea_event_extension.dart +++ b/lib/pangea/events/extensions/pangea_event_extension.dart @@ -59,8 +59,10 @@ extension PangeaEvent on Event { final matrixFile = await downloadAndDecryptAttachment(); - final duration = audioContent?.tryGet('duration') ?? - content.tryGetMap('info')?.tryGet('duration'); + final duration = audioContent?.tryGet(ModelKey.duration) ?? + content + .tryGetMap('info') + ?.tryGet(ModelKey.duration); final waveform = audioContent?.tryGetList('waveform') ?? content diff --git a/lib/pangea/translation/full_text_translation_request_model.dart b/lib/pangea/translation/full_text_translation_request_model.dart index dd5c4cf0e..62375e56c 100644 --- a/lib/pangea/translation/full_text_translation_request_model.dart +++ b/lib/pangea/translation/full_text_translation_request_model.dart @@ -22,7 +22,7 @@ class FullTextTranslationRequestModel { }); Map toJson() => { - "text": text, + ModelKey.text: text, ModelKey.srcLang: srcLang, ModelKey.tgtLang: tgtLang, ModelKey.userL2: userL2, diff --git a/lib/pangea/user/analytics_profile_model.dart b/lib/pangea/user/analytics_profile_model.dart index 36cbb2c45..c7ed6c35a 100644 --- a/lib/pangea/user/analytics_profile_model.dart +++ b/lib/pangea/user/analytics_profile_model.dart @@ -21,12 +21,12 @@ class AnalyticsProfileModel { final profileJson = json[PangeaEventTypes.profileAnalytics]; - final baseLanguage = profileJson[ModelKey.userSourceLanguage] != null - ? PLanguageStore.byLangCode(profileJson[ModelKey.userSourceLanguage]) + final baseLanguage = profileJson[ModelKey.sourceLanguage] != null + ? PLanguageStore.byLangCode(profileJson[ModelKey.sourceLanguage]) : null; - final targetLanguage = profileJson[ModelKey.userTargetLanguage] != null - ? PLanguageStore.byLangCode(profileJson[ModelKey.userTargetLanguage]) + final targetLanguage = profileJson[ModelKey.targetLanguage] != null + ? PLanguageStore.byLangCode(profileJson[ModelKey.targetLanguage]) : null; final languageAnalytics = {}; @@ -59,11 +59,11 @@ class AnalyticsProfileModel { final json = {}; if (targetLanguage != null) { - json[ModelKey.userTargetLanguage] = targetLanguage!.langCodeShort; + json[ModelKey.targetLanguage] = targetLanguage!.langCodeShort; } if (baseLanguage != null) { - json[ModelKey.userSourceLanguage] = baseLanguage!.langCodeShort; + json[ModelKey.sourceLanguage] = baseLanguage!.langCodeShort; } final analytics = {}; diff --git a/lib/pangea/user/user_model.dart b/lib/pangea/user/user_model.dart index c0bd89c63..60ce1873a 100644 --- a/lib/pangea/user/user_model.dart +++ b/lib/pangea/user/user_model.dart @@ -39,8 +39,8 @@ class UserSettings { ? DateTime.parse(json[ModelKey.userCreatedAt]) : null, publicProfile: json[ModelKey.publicProfile], - targetLanguage: json[ModelKey.l2LanguageKey], - sourceLanguage: json[ModelKey.l1LanguageKey], + targetLanguage: json[ModelKey.targetLanguage], + sourceLanguage: json[ModelKey.sourceLanguage], gender: json[ModelKey.userGender] is String ? GenderEnumExtension.fromString( json[ModelKey.userGender], @@ -59,8 +59,8 @@ class UserSettings { data[ModelKey.userDateOfBirth] = dateOfBirth?.toIso8601String(); data[ModelKey.userCreatedAt] = createdAt?.toIso8601String(); data[ModelKey.publicProfile] = publicProfile; - data[ModelKey.l2LanguageKey] = targetLanguage; - data[ModelKey.l1LanguageKey] = sourceLanguage; + data[ModelKey.targetLanguage] = targetLanguage; + data[ModelKey.sourceLanguage] = sourceLanguage; data[ModelKey.userGender] = gender.string; data[ModelKey.userCountry] = country; data[ModelKey.cefrLevel] = cefrLevel.string; @@ -104,10 +104,10 @@ class UserSettings { publicProfile: (accountData[ModelKey.publicProfile] ?.content[ModelKey.publicProfile] as bool?) ?? false, - targetLanguage: accountData[ModelKey.l2LanguageKey] - ?.content[ModelKey.l2LanguageKey] as String?, - sourceLanguage: accountData[ModelKey.l1LanguageKey] - ?.content[ModelKey.l1LanguageKey] as String?, + targetLanguage: accountData[ModelKey.targetLanguage] + ?.content[ModelKey.targetLanguage] as String?, + sourceLanguage: accountData[ModelKey.sourceLanguage] + ?.content[ModelKey.sourceLanguage] as String?, country: accountData[ModelKey.userCountry]?.content[ModelKey.userCountry] as String?, ); @@ -420,10 +420,10 @@ class PangeaProfile { factory PangeaProfile.fromJson(Map json) { final l2 = LanguageModel.codeFromNameOrCode( - json[ModelKey.l2LanguageKey], + json[ModelKey.targetLanguage], ); final l1 = LanguageModel.codeFromNameOrCode( - json[ModelKey.l1LanguageKey], + json[ModelKey.sourceLanguage], ); return PangeaProfile( @@ -442,8 +442,8 @@ class PangeaProfile { data[ModelKey.userCreatedAt] = createdAt; data[ModelKey.userPangeaUserId] = pangeaUserId; data[ModelKey.userDateOfBirth] = dateOfBirth; - data[ModelKey.l2LanguageKey] = targetLanguage; - data[ModelKey.l1LanguageKey] = sourceLanguage; + data[ModelKey.targetLanguage] = targetLanguage; + data[ModelKey.sourceLanguage] = sourceLanguage; data[ModelKey.publicProfile] = publicProfile; data[ModelKey.userCountry] = country; return data; diff --git a/scripts/find_hardcoded_keys.py b/scripts/find_hardcoded_keys.py new file mode 100755 index 000000000..896d2913c --- /dev/null +++ b/scripts/find_hardcoded_keys.py @@ -0,0 +1,142 @@ +#!/usr/bin/env python3 +""" +Script to find hardcoded json keys and replace with ModelKey equivalents + +This script: +1. Reads model_keys.dart and extracts string values +2. Searches the repository for equivalent hardcoded strings +3. Replaces hardcoded strings with ModelKeys + +Usage: + python3 scripts/find_hardcoded_keys.py +""" + +import json +import os +import re +import subprocess +from pathlib import Path +from typing import Set, List + +def extract_pairs(model_keys_path: str) -> List[(str, str)]: + """ + Extract ModelKey names and string values from model_keys.dart file + + ModelKey entries are of the form + static const String name = 'value'; + or + static const String name = "value"; + + Args: + model_keys_path: Path to model_keys.dart + + Returns: + List of ModelKey names and string values, as tuples + """ + pairs = [] + + # Parse each entry for ModelKey name and value + with open(model_keys_path, 'r') as f: + file_content = f.read() + + reg_filter = re.compile("static[\n ]+const[\n ]+String[\n ]+[a-zA-Z0-9]+[\n ]+=[\n ]+[\"\'][a-zA-Z0-9_.]+[\"\'];") + entries = reg_filter.findall(file_content) + + for entry in entries: + parts = entry.strip().split() + + name = parts[3] + value = parts[5] + trimmed_value = value.strip(";\'\"") + pairs.append((name, trimmed_value)) + + return pairs + +def search_and_replace(name: str, value: str, repo_path: str): + """ + Search for a hardcoded ModelKey value in the repository, + using git grep for efficiency. + + Replaces with ModelKey.name if found. + + Args: + name: Name of ModelKey entry + value: ModelKey string value to search for and replace + repo_path: Path to the repository root + + """ + search = "[\"\']" + value + "[\"\']" + replace = "ModelKey." + name + + try: + # Use git grep for fast searching + # Find all files that contain the given hardcoded ModelKey value + result = subprocess.run( + ['git', 'grep', '-l', search], + cwd=repo_path, + capture_output=True, + text=True + ) + + # Remove model_keys.dart and any non-dart files + files = result.stdout.strip().split('\n') + for file in files: + if ("model_keys.dart" not in file) & file.endswith('.dart'): + with open(file, 'r+') as f: + file_content = f.read() + + # Files without ModelKey import statements + # may not need edits - leave to user discretion + import_str = "import 'package:fluffychat/pangea/common/constants/model_keys.dart';" + if import_str not in file_content: + print("The file " + file + " contains the text \"" + value + "\"") + # file_content = import_str + "\n" + file_content + + else: + # Replace instances of hardcoded value in file + file_content = re.sub(search, replace, file_content) + f.seek(0) + f.write(file_content) + f.truncate() + + except subprocess.CalledProcessError: + return False + +def find_hardcoded_values(model_keys_path: str, repo_path: str): + """ + Use helper functions to find hardcoded + ModelKey values and replace them. + + Args: + model_keys_path: Path to model_keys.dart + repo_path: Path to the repository root + """ + # Find list of all ModelKeys to be searched for + # as tuples of (name, value) + pairs = extract_pairs(model_keys_path) + + print(f"Found {len(pairs)} ModelKeys to check.\n") + + print("Searching and replacing hardcoded ModelKey values...") + for pair in pairs: + search_and_replace(pair[0], pair[1], repo_path) + print("Replacement complete") + +def main(): + """Main function to run the hardcoded key replacement function.""" + # Get repository root + repo_path = Path(__file__).parent.parent.absolute() + model_keys_path = repo_path / 'lib' / 'pangea' / 'common' / 'constants' / 'model_keys.dart' + + if not model_keys_path.exists(): + print(f"Error: Could not find {model_keys_path}") + return 1 + + # Find list of all ModelKeys + # and replace any hardcoded values with ModelKey + find_hardcoded_values(str(model_keys_path), str(repo_path)) + + return 0 + +if __name__ == '__main__': + exit(main()) \ No newline at end of file