ember/fluffychat
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Model key cleanup (#4983)

Browse source 
* refactor: Group redundant ModelKey entries

* Add python script to find and replace hardcoded ModelKey values

* Edited Python script to not automatically use ModelKey for files not
already using it

* refactor: Ran script and accepted obvious changes

* rename 'duration' model key

---------

Co-authored-by: ggurdin <ggurdin@gmail.com>
This commit is contained in:
Kelrap 2025-12-31 09:26:06 -05:00 • committed by GitHub
parent 43080978de
commit 711ae38f4a
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
13 changed files with 196 additions and 59 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

18
lib/pangea/activity_planner/activity_plan_model.dart
View file

@ -84,7 +84,7 @@ class ActivityPlanModel {
instructions: json[ModelKey.activityPlanInstructions],
req: req,
title: json[ModelKey.activityPlanTitle],
description: json[ModelKey.activityPlanDescription] ??
description: json[ModelKey.description] ??
json[ModelKey.activityPlanLearningObjective],
learningObjective: json[ModelKey.activityPlanLearningObjective],
vocab: List<Vocab>.from(
@ -93,11 +93,11 @@ class ActivityPlanModel {
endAt: json[ModelKey.activityPlanEndAt] != null
? DateTime.parse(json[ModelKey.activityPlanEndAt])
: null,
duration: json[ModelKey.activityPlanDuration] != null
duration: json[ModelKey.duration] != null
? Duration(
days: json[ModelKey.activityPlanDuration]['days'] ?? 0,
hours: json[ModelKey.activityPlanDuration]['hours'] ?? 0,
minutes: json[ModelKey.activityPlanDuration]['minutes'] ?? 0,
days: json[ModelKey.duration]['days'] ?? 0,
hours: json[ModelKey.duration]['hours'] ?? 0,
minutes: json[ModelKey.duration]['minutes'] ?? 0,
)
: null,
roles: roles,
@ -113,11 +113,11 @@ class ActivityPlanModel {
ModelKey.activityPlanInstructions: instructions,
ModelKey.activityPlanRequest: req.toJson(),
ModelKey.activityPlanTitle: title,
ModelKey.activityPlanDescription: description,
ModelKey.description: description,
ModelKey.activityPlanLearningObjective: learningObjective,
ModelKey.activityPlanVocab: vocab.map((vocab) => vocab.toJson()).toList(),
ModelKey.activityPlanEndAt: endAt?.toIso8601String(),
ModelKey.activityPlanDuration: {
ModelKey.duration: {
'days': duration?.inDays ?? 0,
'hours': duration?.inHours.remainder(24) ?? 0,
'minutes': duration?.inMinutes.remainder(60) ?? 0,
@ -181,7 +181,7 @@ class Vocab {
factory Vocab.fromJson(Map<String, dynamic> json) {
return Vocab(
lemma: json['lemma'],
lemma: json[ModelKey.lemma],
pos: json['pos'],
);
}
@ -203,7 +203,7 @@ class Vocab {
Map<String, dynamic> toJson() {
return {
'lemma': lemma,
ModelKey.lemma: lemma,
'pos': pos,
};
}

8
lib/pangea/activity_planner/activity_plan_request.dart
View file

@ -30,12 +30,12 @@ class ActivityPlanRequest {
Map<String, dynamic> toJson() {
return {
ModelKey.activityRequestTopic: topic,
ModelKey.activityRequestMode: mode,
ModelKey.mode: mode,
ModelKey.activityRequestObjective: objective,
ModelKey.activityRequestMedia: media.string,
ModelKey.activityRequestCefrLevel: cefrLevel.string,
ModelKey.activityRequestLanguageOfInstructions: languageOfInstructions,
ModelKey.activityRequestTargetLanguage: targetLanguage,
ModelKey.targetLanguage: targetLanguage,
ModelKey.activityRequestCount: count,
ModelKey.activityRequestNumberOfParticipants: numberOfParticipants,
ModelKey.activityPlanLocation: location,
@ -45,7 +45,7 @@ class ActivityPlanRequest {
factory ActivityPlanRequest.fromJson(Map<String, dynamic> json) =>
ActivityPlanRequest(
topic: json[ModelKey.activityRequestTopic],
mode: json[ModelKey.activityRequestMode],
mode: json[ModelKey.mode],
objective: json[ModelKey.activityRequestObjective],
media: MediaEnum.nan.fromString(json[ModelKey.activityRequestMedia]),
cefrLevel: json[ModelKey.activityRequestCefrLevel] != null
@ -55,7 +55,7 @@ class ActivityPlanRequest {
: LanguageLevelTypeEnum.a1,
languageOfInstructions:
json[ModelKey.activityRequestLanguageOfInstructions],
targetLanguage: json[ModelKey.activityRequestTargetLanguage],
targetLanguage: json[ModelKey.targetLanguage],
count: json[ModelKey.activityRequestCount],
numberOfParticipants:
json[ModelKey.activityRequestNumberOfParticipants],

4
lib/pangea/choreographer/igc/igc_request_model.dart
View file

@ -25,8 +25,8 @@ class IGCRequestModel {
ModelKey.fullText: fullText,
ModelKey.userL1: userL1,
ModelKey.userL2: userL2,
"enable_it": enableIT,
"enable_igc": enableIGC,
ModelKey.enableIT: enableIT,
ModelKey.enableIGC: enableIGC,
ModelKey.userId: userId,
ModelKey.prevMessages:
jsonEncode(prevMessages.map((x) => x.toJson()).toList()),

8
lib/pangea/choreographer/igc/igc_response_model.dart
View file

@ -36,8 +36,8 @@ class IGCResponseModel {
fullTextCorrection: json["full_text_correction"],
userL1: json[ModelKey.userL1],
userL2: json[ModelKey.userL2],
enableIT: json["enable_it"],
enableIGC: json["enable_igc"],
enableIT: json[ModelKey.enableIT],
enableIGC: json[ModelKey.enableIGC],
);
}
@ -47,7 +47,7 @@ class IGCResponseModel {
"matches": matches.map((e) => e.toJson()).toList(),
ModelKey.userL1: userL1,
ModelKey.userL2: userL2,
"enable_it": enableIT,
"enable_igc": enableIGC,
ModelKey.enableIT: enableIT,
ModelKey.enableIGC: enableIGC,
};
}

4
lib/pangea/choreographer/igc/span_data_request.dart
View file

@ -19,8 +19,8 @@ class SpanDetailsRequest {
Map<String, dynamic> toJson() => {
ModelKey.userL1: userL1,
ModelKey.userL2: userL2,
"enable_it": enableIT,
"enable_igc": enableIGC,
ModelKey.enableIT: enableIT,
ModelKey.enableIGC: enableIGC,
'span': span.toJson(),
};

8
lib/pangea/choreographer/it/it_request_model.dart
View file

@ -22,11 +22,11 @@ class ITRequestModel {
});
factory ITRequestModel.fromJson(json) => ITRequestModel(
text: json['text'],
text: json[ModelKey.text],
customInput: json['custom_input'],
sourceLangCode: json[ModelKey.srcLang],
targetLangCode: json[ModelKey.tgtLang],
goldTranslation: json['gold_translation'],
goldTranslation: json[ModelKey.goldTranslation],
goldContinuances: json['gold_continuances'] != null
? (json['gold_continuances'])
.map((e) => ContinuanceModel.fromJson(e))
@ -35,11 +35,11 @@ class ITRequestModel {
);
Map<String, dynamic> toJson() => {
'text': text,
ModelKey.text: text,
'custom_input': customInput,
ModelKey.srcLang: sourceLangCode,
ModelKey.tgtLang: targetLangCode,
'gold_translation': goldTranslation,
ModelKey.goldTranslation: goldTranslation,
'gold_continuances': goldContinuances != null
? List.from(goldContinuances!.map((e) => e.toJson()))
: null,

13
lib/pangea/common/constants/model_keys.dart
View file

@ -7,14 +7,10 @@ class ModelKey {
static const String userCreatedAt = 'created_at';
static const String userPangeaUserId = 'pangea_user_id';
static const String userDateOfBirth = 'date_of_birth';
static const String userTargetLanguage = 'target_language';
static const String userSourceLanguage = 'source_language';
static const String userSpeaks = 'speaks';
static const String userCountry = 'country';
static const String hasJoinedHelpSpace = 'has_joined_help_space';
static const String userInterests = 'interests';
static const String l2LanguageKey = 'target_language';
static const String l1LanguageKey = 'source_language';
static const String publicProfile = 'public_profile';
static const String userId = 'user_id';
static const String toolSettings = 'tool_settings';
@ -26,10 +22,8 @@ class ModelKey {
static const String itAutoPlay = 'autoPlayIT';
static const String clientClassCity = "city";
static const String clientClassCountry = "country";
static const String clientClassDominantLanguage = "dominantLanguage";
static const String clientClassTargetLanguage = "targetLanguage";
static const String clientClassDescription = "description";
static const String clientLanguageLevel = "languageLevel";
static const String clientSchool = "schoolName";
@ -136,6 +130,7 @@ class ModelKey {
"text_adventure_game_master_instructions";
static const String targetLanguage = "target_language";
static const String sourceLanguage = "source_language";
static const String targetVoice = "target_voice";
static const String prevEventId = "prev_event_id";
@ -160,7 +155,8 @@ class ModelKey {
// activity plan
static const String activityPlanRequest = "req";
static const String activityPlanTitle = "title";
static const String activityPlanDescription = "description";
static const String description = "description";
static const String duration = "duration";
static const String activityPlanLocation = "location";
static const String activityPlanLearningObjective = "learning_objective";
static const String activityPlanInstructions = "instructions";
@ -168,17 +164,14 @@ class ModelKey {
static const String activityPlanImageURL = "image_url";
static const String activityId = "activity_id";
static const String activityPlanEndAt = "end_at";
static const String activityPlanDuration = "duration";
static const String activityPlanTopicId = "topic_id";
static const String activityRequestTopic = "topic";
static const String activityRequestMode = "mode";
static const String activityRequestObjective = "objective";
static const String activityRequestMedia = "media";
static const String activityRequestCefrLevel = "activity_cefr_level";
static const String activityRequestLanguageOfInstructions =
"language_of_instructions";
static const String activityRequestTargetLanguage = "target_language";
static const String activityRequestCount = "count";
static const String activityRequestNumberOfParticipants =
"number_of_participants";

6
lib/pangea/events/event_wrappers/pangea_message_event.dart
View file

@ -293,7 +293,7 @@ class PangeaMessageEvent {
Event? getTextToSpeechLocal(String langCode, String text) {
for (final audio in allAudio) {
final dataMap = audio.content.tryGetMap(ModelKey.transcription);
if (dataMap == null || !dataMap.containsKey('tokens')) continue;
if (dataMap == null || !dataMap.containsKey(ModelKey.tokens)) continue;
try {
final PangeaAudioEventData audioData = PangeaAudioEventData.fromJson(
@ -413,11 +413,11 @@ class PangeaMessageEvent {
extraContent: {
'info': {
...file.info,
'duration': response.durationMillis,
ModelKey.duration: response.durationMillis,
},
'org.matrix.msc3245.voice': {},
'org.matrix.msc1767.audio': {
'duration': response.durationMillis,
ModelKey.duration: response.durationMillis,
'waveform': response.waveform,
},
ModelKey.transcription: response

6
lib/pangea/events/extensions/pangea_event_extension.dart
View file

@ -59,8 +59,10 @@ extension PangeaEvent on Event {
final matrixFile = await downloadAndDecryptAttachment();
final duration = audioContent?.tryGet<int>('duration') ??
content.tryGetMap<String, dynamic>('info')?.tryGet<int>('duration');
final duration = audioContent?.tryGet<int>(ModelKey.duration) ??
content
.tryGetMap<String, dynamic>('info')
?.tryGet<int>(ModelKey.duration);
final waveform = audioContent?.tryGetList<int>('waveform') ??
content

2
lib/pangea/translation/full_text_translation_request_model.dart
View file

@ -22,7 +22,7 @@ class FullTextTranslationRequestModel {
});
Map<String, dynamic> toJson() => {
"text": text,
ModelKey.text: text,
ModelKey.srcLang: srcLang,
ModelKey.tgtLang: tgtLang,
ModelKey.userL2: userL2,

12
lib/pangea/user/analytics_profile_model.dart
View file

@ -21,12 +21,12 @@ class AnalyticsProfileModel {
final profileJson = json[PangeaEventTypes.profileAnalytics];
final baseLanguage = profileJson[ModelKey.userSourceLanguage] != null
? PLanguageStore.byLangCode(profileJson[ModelKey.userSourceLanguage])
final baseLanguage = profileJson[ModelKey.sourceLanguage] != null
? PLanguageStore.byLangCode(profileJson[ModelKey.sourceLanguage])
: null;
final targetLanguage = profileJson[ModelKey.userTargetLanguage] != null
? PLanguageStore.byLangCode(profileJson[ModelKey.userTargetLanguage])
final targetLanguage = profileJson[ModelKey.targetLanguage] != null
? PLanguageStore.byLangCode(profileJson[ModelKey.targetLanguage])
: null;
final languageAnalytics = <LanguageModel, LanguageAnalyticsProfileEntry>{};
@ -59,11 +59,11 @@ class AnalyticsProfileModel {
final json = <String, dynamic>{};
if (targetLanguage != null) {
json[ModelKey.userTargetLanguage] = targetLanguage!.langCodeShort;
json[ModelKey.targetLanguage] = targetLanguage!.langCodeShort;
}
if (baseLanguage != null) {
json[ModelKey.userSourceLanguage] = baseLanguage!.langCodeShort;
json[ModelKey.sourceLanguage] = baseLanguage!.langCodeShort;
}
final analytics = {};

24
lib/pangea/user/user_model.dart
View file

@ -39,8 +39,8 @@ class UserSettings {
? DateTime.parse(json[ModelKey.userCreatedAt])
: null,
publicProfile: json[ModelKey.publicProfile],
targetLanguage: json[ModelKey.l2LanguageKey],
sourceLanguage: json[ModelKey.l1LanguageKey],
targetLanguage: json[ModelKey.targetLanguage],
sourceLanguage: json[ModelKey.sourceLanguage],
gender: json[ModelKey.userGender] is String
? GenderEnumExtension.fromString(
json[ModelKey.userGender],
@ -59,8 +59,8 @@ class UserSettings {
data[ModelKey.userDateOfBirth] = dateOfBirth?.toIso8601String();
data[ModelKey.userCreatedAt] = createdAt?.toIso8601String();
data[ModelKey.publicProfile] = publicProfile;
data[ModelKey.l2LanguageKey] = targetLanguage;
data[ModelKey.l1LanguageKey] = sourceLanguage;
data[ModelKey.targetLanguage] = targetLanguage;
data[ModelKey.sourceLanguage] = sourceLanguage;
data[ModelKey.userGender] = gender.string;
data[ModelKey.userCountry] = country;
data[ModelKey.cefrLevel] = cefrLevel.string;
@ -104,10 +104,10 @@ class UserSettings {
publicProfile: (accountData[ModelKey.publicProfile]
?.content[ModelKey.publicProfile] as bool?) ??
false,
targetLanguage: accountData[ModelKey.l2LanguageKey]
?.content[ModelKey.l2LanguageKey] as String?,
sourceLanguage: accountData[ModelKey.l1LanguageKey]
?.content[ModelKey.l1LanguageKey] as String?,
targetLanguage: accountData[ModelKey.targetLanguage]
?.content[ModelKey.targetLanguage] as String?,
sourceLanguage: accountData[ModelKey.sourceLanguage]
?.content[ModelKey.sourceLanguage] as String?,
country: accountData[ModelKey.userCountry]?.content[ModelKey.userCountry]
as String?,
);
@ -420,10 +420,10 @@ class PangeaProfile {
factory PangeaProfile.fromJson(Map<String, dynamic> json) {
final l2 = LanguageModel.codeFromNameOrCode(
json[ModelKey.l2LanguageKey],
json[ModelKey.targetLanguage],
);
final l1 = LanguageModel.codeFromNameOrCode(
json[ModelKey.l1LanguageKey],
json[ModelKey.sourceLanguage],
);
return PangeaProfile(
@ -442,8 +442,8 @@ class PangeaProfile {
data[ModelKey.userCreatedAt] = createdAt;
data[ModelKey.userPangeaUserId] = pangeaUserId;
data[ModelKey.userDateOfBirth] = dateOfBirth;
data[ModelKey.l2LanguageKey] = targetLanguage;
data[ModelKey.l1LanguageKey] = sourceLanguage;
data[ModelKey.targetLanguage] = targetLanguage;
data[ModelKey.sourceLanguage] = sourceLanguage;
data[ModelKey.publicProfile] = publicProfile;
data[ModelKey.userCountry] = country;
return data;

142
scripts/find_hardcoded_keys.py Executable file
View file

@ -0,0 +1,142 @@
#!/usr/bin/env python3
"""
Script to find hardcoded json keys and replace with ModelKey equivalents
This script:
1. Reads model_keys.dart and extracts string values
2. Searches the repository for equivalent hardcoded strings
3. Replaces hardcoded strings with ModelKeys
Usage:
python3 scripts/find_hardcoded_keys.py
"""
import json
import os
import re
import subprocess
from pathlib import Path
from typing import Set, List
def extract_pairs(model_keys_path: str) -> List[(str, str)]:
"""
Extract ModelKey names and string values from model_keys.dart file
ModelKey entries are of the form
static const String name = 'value';
or
static const String name = "value";
Args:
model_keys_path: Path to model_keys.dart
Returns:
List of ModelKey names and string values, as tuples
"""
pairs = []
# Parse each entry for ModelKey name and value
with open(model_keys_path, 'r') as f:
file_content = f.read()
reg_filter = re.compile("static[\n ]+const[\n ]+String[\n ]+[a-zA-Z0-9]+[\n ]+=[\n ]+[\"\'][a-zA-Z0-9_.]+[\"\'];")
entries = reg_filter.findall(file_content)
for entry in entries:
parts = entry.strip().split()
name = parts[3]
value = parts[5]
trimmed_value = value.strip(";\'\"")
pairs.append((name, trimmed_value))
return pairs
def search_and_replace(name: str, value: str, repo_path: str):
"""
Search for a hardcoded ModelKey value in the repository,
using git grep for efficiency.
Replaces with ModelKey.name if found.
Args:
name: Name of ModelKey entry
value: ModelKey string value to search for and replace
repo_path: Path to the repository root
"""
search = "[\"\']" + value + "[\"\']"
replace = "ModelKey." + name
try:
# Use git grep for fast searching
# Find all files that contain the given hardcoded ModelKey value
result = subprocess.run(
['git', 'grep', '-l', search],
cwd=repo_path,
capture_output=True,
text=True
)
# Remove model_keys.dart and any non-dart files
files = result.stdout.strip().split('\n')
for file in files:
if ("model_keys.dart" not in file) & file.endswith('.dart'):
with open(file, 'r+') as f:
file_content = f.read()
# Files without ModelKey import statements
# may not need edits - leave to user discretion
import_str = "import 'package:fluffychat/pangea/common/constants/model_keys.dart';"
if import_str not in file_content:
print("The file " + file + " contains the text \"" + value + "\"")
# file_content = import_str + "\n" + file_content
else:
# Replace instances of hardcoded value in file
file_content = re.sub(search, replace, file_content)
f.seek(0)
f.write(file_content)
f.truncate()
except subprocess.CalledProcessError:
return False
def find_hardcoded_values(model_keys_path: str, repo_path: str):
"""
Use helper functions to find hardcoded
ModelKey values and replace them.
Args:
model_keys_path: Path to model_keys.dart
repo_path: Path to the repository root
"""
# Find list of all ModelKeys to be searched for
# as tuples of (name, value)
pairs = extract_pairs(model_keys_path)
print(f"Found {len(pairs)} ModelKeys to check.\n")
print("Searching and replacing hardcoded ModelKey values...")
for pair in pairs:
search_and_replace(pair[0], pair[1], repo_path)
print("Replacement complete")
def main():
"""Main function to run the hardcoded key replacement function."""
# Get repository root
repo_path = Path(__file__).parent.parent.absolute()
model_keys_path = repo_path / 'lib' / 'pangea' / 'common' / 'constants' / 'model_keys.dart'
if not model_keys_path.exists():
print(f"Error: Could not find {model_keys_path}")
return 1
# Find list of all ModelKeys
# and replace any hardcoded values with ModelKey
find_hardcoded_values(str(model_keys_path), str(repo_path))
return 0
if __name__ == '__main__':
exit(main())