ember/fluffychat
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

feat: leave analytics rooms after extracting data, use generator function to batch rooms in download (#1478)

Browse source
This commit is contained in:
ggurdin 2025-01-16 12:42:24 -05:00 • committed by GitHub
parent 77c4f711b0
commit 5588d8ec16
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
4 changed files with 213 additions and 121 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

230
lib/pangea/analytics/extensions/room_analytics_extension.dart
View file

@ -1,100 +1,176 @@
part of "../../extensions/pangea_room_extension.dart";
extension AnalyticsRoomExtension on Room {
Future<List<SpaceRoomsChunk>> _getFullSpaceHierarchy() async {
/// Get next n analytics rooms via the space hierarchy
/// If joined
/// If not in target language
/// If not created by user, leave
/// Else, add to list
/// Else
/// If room name does not match L2, skip
/// Join and wait for room in sync.
/// Repeat the same procedure as above.
///
/// If not n analytics rooms in list, and nextBatch != null, repeat the above
/// procedure with nextBatch until n analytics rooms are found or nextBatch == null
///
/// Yield this list of rooms.
/// Once analytics have been retrieved, leave analytics rooms not created by self.
Stream<List<Room>> getNextAnalyticsRoomBatch(String userL2) async* {
final List<SpaceRoomsChunk> rooms = [];
String? nextBatch;
int spaceHierarchyCalls = 0;
int callsToServer = 0;
while (spaceHierarchyCalls <= 5 &&
(nextBatch != null || spaceHierarchyCalls == 0)) {
spaceHierarchyCalls++;
final resp = await _getNextBatch(nextBatch);
callsToServer++;
if (resp == null) return;
rooms.addAll(resp.rooms);
nextBatch = resp.nextBatch;
final List<Room> roomsBatch = [];
while (rooms.isNotEmpty) {
// prevent rate-limiting
if (callsToServer >= 5) {
callsToServer = 0;
await Future.delayed(const Duration(milliseconds: 7500));
}
final nextRoomChunk = rooms.removeAt(0);
if (nextRoomChunk.roomType != PangeaRoomTypes.analytics) {
continue;
}
final matchingRoom = client.rooms.firstWhereOrNull(
(r) => r.id == nextRoomChunk.roomId,
);
final (analyticsRoom, calls) = matchingRoom != null
? await _handleJoinedAnalyticsRoom(matchingRoom, userL2)
: await _handleUnjoinedAnalyticsRoom(nextRoomChunk, userL2);
callsToServer += calls;
if (analyticsRoom == null) continue;
roomsBatch.add(analyticsRoom);
if (roomsBatch.length >= 5) {
final roomsBatchCopy = List<Room>.from(roomsBatch);
roomsBatch.clear();
yield roomsBatchCopy;
}
}
yield roomsBatch;
}
}
/// Return analytics room, given unjoined member of space hierarchy,
/// if should get analytics for that room, and number of call made
/// to the server to help prevent rate-limiting
Future<(Room?, int)> _handleUnjoinedAnalyticsRoom(
SpaceRoomsChunk chunk,
String l2,
) async {
int callsToServer = 0;
final nameParts = chunk.name?.split(" ");
if (nameParts != null && nameParts.length >= 2) {
final roomLangCode = nameParts[1];
if (roomLangCode != l2) return (null, callsToServer);
}
Room? analyticsRoom = await _joinAnalyticsRoomChunk(chunk);
callsToServer++;
if (analyticsRoom == null) return (null, callsToServer);
final (room, calls) = await _handleJoinedAnalyticsRoom(analyticsRoom, l2);
analyticsRoom = room;
callsToServer += calls;
return (analyticsRoom, callsToServer);
}
/// Return analytics room if should add to returned list
/// and the number of calls made to the server (used to prevent rate-limiting)
Future<(Room?, int)> _handleJoinedAnalyticsRoom(
Room analyticsRoom,
String l2,
) async {
if (client.userID == null) return (null, 0);
if (analyticsRoom.madeForLang != l2) {
await _leaveNonTargetAnalyticsRoom(analyticsRoom, l2);
return (null, 1);
}
return (analyticsRoom, 0);
}
Future<Room?> _joinAnalyticsRoomChunk(
SpaceRoomsChunk chunk,
) async {
final matchingRoom = client.rooms.firstWhereOrNull(
(r) => r.id == chunk.roomId,
);
if (matchingRoom != null) return matchingRoom;
try {
final syncFuture = client.waitForRoomInSync(chunk.roomId, join: true);
await client.joinRoom(chunk.roomId);
await syncFuture;
return client.getRoomById(chunk.roomId);
} catch (e, s) {
ErrorHandler.logError(
e: e,
s: s,
data: {
"roomID": chunk.roomId,
},
);
return null;
}
}
Future<void> _leaveNonTargetAnalyticsRoom(Room room, String userL2) async {
if (client.userID == null ||
room.isMadeByUser(client.userID!) ||
room.madeForLang == userL2) {
return;
}
try {
await room.leave();
} catch (e, s) {
ErrorHandler.logError(
e: e,
s: s,
data: {
"roomID": room.id,
},
);
}
}
Future<GetSpaceHierarchyResponse?> _getNextBatch(String? nextBatch) async {
try {
final resp = await client.getSpaceHierarchy(
id,
from: nextBatch,
limit: 100,
maxDepth: 1,
);
rooms.addAll(resp.rooms);
nextBatch = resp.nextBatch;
return resp;
} catch (e, s) {
ErrorHandler.logError(
e: e,
s: s,
data: {
"spaceID": id,
"nextBatch": nextBatch,
},
);
return rooms;
}
int tries = 0;
while (nextBatch != null && tries <= 5) {
GetSpaceHierarchyResponse nextResp;
try {
nextResp = await client.getSpaceHierarchy(
id,
from: nextBatch,
limit: 100,
maxDepth: 1,
);
rooms.addAll(nextResp.rooms);
} catch (e, s) {
ErrorHandler.logError(
e: e,
s: s,
data: {
"spaceID": id,
},
);
break;
}
nextBatch = nextResp.nextBatch;
tries++;
}
return rooms;
}
Future<void> _joinAnalyticsRooms() async {
final List<SpaceRoomsChunk> rooms = await _getFullSpaceHierarchy();
final unjoinedAnalyticsRooms = rooms.where(
(room) {
if (room.roomType != PangeaRoomTypes.analytics) return false;
final matchingRoom = client.rooms.firstWhereOrNull(
(r) => r.id == room.roomId,
);
return matchingRoom == null ||
matchingRoom.membership != Membership.join;
},
).toList();
const batchSize = 5;
int batchNum = 0;
while (batchSize * batchNum < unjoinedAnalyticsRooms.length) {
final batch =
unjoinedAnalyticsRooms.sublist(batchSize * batchNum).take(batchSize);
batchNum++;
for (final analyticsRoom in batch) {
try {
final syncFuture =
client.waitForRoomInSync(analyticsRoom.roomId, join: true);
await client.joinRoom(analyticsRoom.roomId);
await syncFuture;
} catch (e, s) {
ErrorHandler.logError(
e: e,
s: s,
data: {
"spaceID": id,
"roomID": analyticsRoom.roomId,
},
);
}
}
if (batchSize * batchNum < unjoinedAnalyticsRooms.length) {
await Future.delayed(const Duration(milliseconds: 7500));
}
return null;
}
}

2
lib/pangea/extensions/pangea_room_extension.dart
View file

@ -44,8 +44,6 @@ part "room_user_permissions_extension.dart";
extension PangeaRoom on Room {
// analytics
Future<void> joinAnalyticsRooms() async => await _joinAnalyticsRooms();
Future<DateTime?> analyticsLastUpdated(String userId) async {
return await _analyticsLastUpdated(userId);
}

100
lib/pangea/spaces/widgets/download_analytics_dialog.dart
View file

@ -1,6 +1,5 @@
import 'package:flutter/material.dart';
import 'package:collection/collection.dart';
import 'package:csv/csv.dart';
import 'package:excel/excel.dart';
import 'package:flutter_gen/gen_l10n/l10n.dart';
@ -34,10 +33,9 @@ class DownloadAnalyticsDialog extends StatefulWidget {
class DownloadAnalyticsDialogState extends State<DownloadAnalyticsDialog> {
bool _initialized = false;
bool _downloaded = false;
bool _joiningRooms = false;
bool _downloading = false;
bool get _loading => _joiningRooms || _downloading || !_initialized;
bool get _loading => _downloading || !_initialized;
String? _error;
@ -65,6 +63,9 @@ class DownloadAnalyticsDialogState extends State<DownloadAnalyticsDialog> {
},
);
} finally {
_downloadStatuses = Map.fromEntries(
_usersToDownload.map((user) => MapEntry(user.id, 0)),
);
if (mounted) setState(() => _initialized = true);
}
}
@ -78,7 +79,6 @@ class DownloadAnalyticsDialogState extends State<DownloadAnalyticsDialog> {
void _clean() {
_error = null;
_joiningRooms = false;
_downloading = false;
_downloaded = false;
_downloadStatuses = Map.fromEntries(
@ -88,7 +88,11 @@ class DownloadAnalyticsDialogState extends State<DownloadAnalyticsDialog> {
List<User> get _usersToDownload => widget.space
.getParticipants()
.where((member) => member.id != BotName.byEnvironment)
.where(
(member) =>
member.id != BotName.byEnvironment &&
member.membership == Membership.join,
)
.toList();
Color _downloadStatusColor(String userID) {
@ -100,38 +104,41 @@ class DownloadAnalyticsDialogState extends State<DownloadAnalyticsDialog> {
}
String? get _statusText {
if (_joiningRooms) return L10n.of(context).accessingMemberAnalytics;
if (_downloading) return L10n.of(context).downloading;
if (_downloaded) return L10n.of(context).downloadComplete;
return null;
}
Room? _userAnalyticsRoom(String userID) {
final rooms = widget.space.client.rooms;
final l2 = MatrixState.pangeaController.languageController.userL2?.langCode;
if (l2 == null) return null;
return rooms.firstWhereOrNull((room) {
return room.isAnalyticsRoomOfUser(userID) && room.isMadeForLang(l2);
});
}
String? get userL2 =>
MatrixState.pangeaController.languageController.userL2?.langCode;
Future<void> _runDownload() async {
try {
if (!mounted) return;
if (!mounted || userL2 == null) return;
setState(() {
_error = null;
_joiningRooms = true;
_downloading = true;
});
await widget.space.joinAnalyticsRooms();
if (mounted) {
setState(() {
_joiningRooms = false;
_downloading = true;
});
final List<AnalyticsSummaryModel> summaries = [];
await for (final batch
in widget.space.getNextAnalyticsRoomBatch(userL2!)) {
if (batch.isEmpty) continue;
final List<AnalyticsSummaryModel?> batchSummaries = await Future.wait(
batch.map((r) => _getAnalyticsModel(r)),
);
summaries.addAll(batchSummaries.whereType<AnalyticsSummaryModel>());
}
await _downloadSpaceAnalytics();
for (final userID in _downloadStatuses.keys) {
if (_downloadStatuses[userID] == 0) {
_downloadStatuses[userID] = -1;
summaries.add(AnalyticsSummaryModel.emptyModel(userID));
}
}
await _downloadSpaceAnalytics(summaries);
if (mounted) {
setState(() {
_downloading = false;
@ -153,18 +160,12 @@ class DownloadAnalyticsDialogState extends State<DownloadAnalyticsDialog> {
}
}
Future<void> _downloadSpaceAnalytics() async {
final l2 = MatrixState.pangeaController.languageController.userL2?.langCode;
if (l2 == null) return;
final List<AnalyticsSummaryModel?> summaries = await Future.wait(
_usersToDownload.map((user) => _getUserAnalyticsModel(user.id)),
);
final allSummaries = summaries.whereType<AnalyticsSummaryModel>().toList();
Future<void> _downloadSpaceAnalytics(
List<AnalyticsSummaryModel> summaries,
) async {
final content = _downloadType == DownloadType.xlsx
? _getExcelFileContent(allSummaries)
: _getCSVFileContent(allSummaries);
? _getExcelFileContent(summaries)
: _getCSVFileContent(summaries);
final fileName =
"analytics_${widget.space.name}_${DateTime.now().toIso8601String()}.${_downloadType == DownloadType.xlsx ? 'xlsx' : 'csv'}";
@ -176,13 +177,16 @@ class DownloadAnalyticsDialogState extends State<DownloadAnalyticsDialog> {
);
}
Future<AnalyticsSummaryModel?> _getUserAnalyticsModel(String userID) async {
Future<AnalyticsSummaryModel?> _getAnalyticsModel(Room analyticsRoom) async {
final String? userID = analyticsRoom.creatorId;
if (userID == null) return null;
AnalyticsSummaryModel? summary;
try {
final userAnalyticsRoom = _userAnalyticsRoom(userID);
_downloadStatuses[userID] = userAnalyticsRoom != null ? 1 : -1;
_downloadStatuses[userID] = 1;
if (mounted) setState(() {});
final constructEvents = await userAnalyticsRoom?.getAnalyticsEvents(
final constructEvents = await analyticsRoom.getAnalyticsEvents(
userId: userID,
);
@ -197,14 +201,13 @@ class DownloadAnalyticsDialogState extends State<DownloadAnalyticsDialog> {
}
final constructs = ConstructListModel(uses: uses);
final summary = AnalyticsSummaryModel.fromConstructListModel(
summary = AnalyticsSummaryModel.fromConstructListModel(
userID,
constructs,
getCopy,
context,
);
if (mounted) setState(() => _downloadStatuses[userID] = 2);
return summary;
} catch (e, s) {
ErrorHandler.logError(
e: e,
@ -215,8 +218,23 @@ class DownloadAnalyticsDialogState extends State<DownloadAnalyticsDialog> {
},
);
if (mounted) setState(() => _downloadStatuses[userID] = -2);
} finally {
if (userID != widget.space.client.userID) {
try {
await analyticsRoom.leave();
} catch (e, s) {
ErrorHandler.logError(
e: e,
s: s,
data: {
"spaceID": widget.space.id,
"userID": userID,
},
);
}
}
}
return null;
return summary;
}
List<CellValue> _formatExcelRow(

2
pubspec.lock
View file

@ -1523,7 +1523,7 @@ packages:
description:
path: "."
ref: main
resolved-ref: f03945433cbcf7ffc33b521a9190630d8bb54513
resolved-ref: "16a089ad229671f2367c8927b9edae776004a3ae"
url: "https://github.com/pangeachat/matrix-dart-sdk.git"
source: git
version: "0.36.0"