diff --git a/Cargo.lock b/Cargo.lock index 504848b7..eef085c2 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1216,7 +1216,6 @@ dependencies = [ "log", "loole", "lru-cache", - "peg", "rand 0.8.5", "recaptcha-verify", "regex", @@ -5136,6 +5135,15 @@ version = "1.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6ce2be8dc25455e1f91df71bfa12ad37d7af1092ae736f3a6cd0e37bc7810596" +[[package]] +name = "stitcher" +version = "0.5.3" +dependencies = [ + "indexmap", + "itertools 0.14.0", + "peg", +] + [[package]] name = "strict" version = "0.2.0" diff --git a/src/service/Cargo.toml b/src/service/Cargo.toml index 26f45f58..b190473e 100644 --- a/src/service/Cargo.toml +++ b/src/service/Cargo.toml @@ -126,6 +126,3 @@ sd-notify.optional = true [lints] workspace = true - -[dev-dependencies] -peg = "0.8.5" diff --git a/src/service/rooms/timeline/mod.rs b/src/service/rooms/timeline/mod.rs index 393143fd..a35b502c 100644 --- a/src/service/rooms/timeline/mod.rs +++ b/src/service/rooms/timeline/mod.rs @@ -4,7 +4,6 @@ mod build; mod create; mod data; mod redact; -mod stitcher; use std::{fmt::Write, sync::Arc}; diff --git a/src/stitcher/Cargo.toml b/src/stitcher/Cargo.toml new file mode 100644 index 00000000..0259e3ce --- /dev/null +++ b/src/stitcher/Cargo.toml @@ -0,0 +1,18 @@ +[package] +name = "stitcher" +description = "An implementation of stitched ordering (https://codeberg.org/andybalaam/stitched-order)" +edition.workspace = true +license.workspace = true +readme.workspace = true +repository.workspace = true +version.workspace = true + +[lib] +path = "mod.rs" + +[dependencies] +indexmap.workspace = true +itertools.workspace = true + +[dev-dependencies] +peg = "0.8.5" diff --git a/src/service/rooms/timeline/stitcher/algorithm.rs b/src/stitcher/algorithm.rs similarity index 93% rename from src/service/rooms/timeline/stitcher/algorithm.rs rename to src/stitcher/algorithm.rs index e0517c1b..810bd953 100644 --- a/src/service/rooms/timeline/stitcher/algorithm.rs +++ b/src/stitcher/algorithm.rs @@ -7,7 +7,7 @@ use super::{Batch, Gap, OrderKey, StitchedItem, StitcherBackend}; /// Updates to a gap in the stitched order. #[derive(Debug)] -pub(super) struct GapUpdate<'id, K: OrderKey> { +pub struct GapUpdate<'id, K: OrderKey> { /// The opaque key of the gap to update. pub key: K, /// The new contents of the gap. If this is empty, the gap should be @@ -20,7 +20,7 @@ pub(super) struct GapUpdate<'id, K: OrderKey> { /// Updates to the stitched order. #[derive(Debug)] -pub(super) struct OrderUpdates<'id, K: OrderKey> { +pub struct OrderUpdates<'id, K: OrderKey> { /// Updates to individual gaps. The items inserted by these updates _should /// not_ be synchronized to clients. pub gap_updates: Vec>, @@ -34,18 +34,18 @@ pub(super) struct OrderUpdates<'id, K: OrderKey> { /// The stitcher, which implements the stitched ordering algorithm. /// Its primary method is [`Stitcher::stitch`]. -pub(super) struct Stitcher<'backend, B: StitcherBackend> { +pub struct Stitcher<'backend, B: StitcherBackend> { backend: &'backend B, } impl Stitcher<'_, B> { /// Create a new [`Stitcher`] given a [`StitcherBackend`]. - pub(super) fn new(backend: &B) -> Stitcher<'_, B> { Stitcher { backend } } + pub fn new(backend: &B) -> Stitcher<'_, B> { Stitcher { backend } } /// Given a [`Batch`], compute the [`OrderUpdates`] which should be made to /// the stitched order to incorporate that batch. It is the responsibility /// of the caller to apply the updates. - pub(super) fn stitch<'id>(&self, batch: &Batch<'id>) -> OrderUpdates<'id, B::Key> { + pub fn stitch<'id>(&self, batch: &Batch<'id>) -> OrderUpdates<'id, B::Key> { let mut gap_updates = Vec::new(); let mut events_added_to_gaps: HashSet<&'id str> = HashSet::new(); diff --git a/src/service/rooms/timeline/stitcher/mod.rs b/src/stitcher/mod.rs similarity index 91% rename from src/service/rooms/timeline/stitcher/mod.rs rename to src/stitcher/mod.rs index 73b8285a..2f0bcd22 100644 --- a/src/service/rooms/timeline/stitcher/mod.rs +++ b/src/stitcher/mod.rs @@ -2,16 +2,17 @@ use std::{cmp::Ordering, collections::HashSet}; use indexmap::IndexMap; -pub(super) mod algorithm; +pub mod algorithm; #[cfg(test)] mod test; +pub use algorithm::*; /// A gap in the stitched order. -pub(super) type Gap = HashSet; +pub type Gap = HashSet; /// An item in the stitched order. #[derive(Debug)] -pub(super) enum StitchedItem<'id> { +pub enum StitchedItem<'id> { /// A single event. Event(&'id str), /// A gap representing one or more missing events. @@ -20,12 +21,12 @@ pub(super) enum StitchedItem<'id> { /// An opaque key returned by a [`StitcherBackend`] to identify an item in its /// order. -pub(super) trait OrderKey: Eq + Clone {} +pub trait OrderKey: Eq + Clone {} impl OrderKey for T {} /// A trait providing read-only access to an existing stitched order. -pub(super) trait StitcherBackend { +pub trait StitcherBackend { type Key: OrderKey; /// Return all gaps containing one or more events listed in `events`. @@ -39,7 +40,7 @@ pub(super) trait StitcherBackend { } /// An ordered map from an event ID to its `prev_events`. -pub(super) type EventEdges<'id> = IndexMap<&'id str, HashSet<&'id str>>; +pub type EventEdges<'id> = IndexMap<&'id str, HashSet<&'id str>>; /// Information about the `prev_events` of an event. /// This struct does not store the ID of the event itself. @@ -56,17 +57,17 @@ struct EventPredecessors<'id> { /// A batch of events to be inserted into the stitched order. #[derive(Debug)] -pub(super) struct Batch<'id> { +pub struct Batch<'id> { events: IndexMap<&'id str, EventPredecessors<'id>>, } impl<'id> Batch<'id> { /// Create a new [`Batch`] from an [`EventEdges`]. - pub(super) fn from_edges<'edges>(edges: &EventEdges<'edges>) -> Batch<'edges> { + pub fn from_edges<'edges>(edges: &EventEdges<'edges>) -> Batch<'edges> { let mut events = IndexMap::new(); for (event, prev_events) in edges { - let predecessor_set = Self::find_predecessor_set(event, &edges); + let predecessor_set = Self::find_predecessor_set(event, edges); events.insert(*event, EventPredecessors { prev_events: prev_events.clone(), diff --git a/src/service/rooms/timeline/stitcher/test/mod.rs b/src/stitcher/test/mod.rs similarity index 98% rename from src/service/rooms/timeline/stitcher/test/mod.rs rename to src/stitcher/test/mod.rs index e9798463..ca065def 100644 --- a/src/service/rooms/timeline/stitcher/test/mod.rs +++ b/src/stitcher/test/mod.rs @@ -3,7 +3,6 @@ use std::sync::atomic::{AtomicU64, Ordering}; use itertools::Itertools; use super::{algorithm::*, *}; -use crate::rooms::timeline::stitcher::algorithm::Stitcher; mod parser; diff --git a/src/service/rooms/timeline/stitcher/test/parser.rs b/src/stitcher/test/parser.rs similarity index 98% rename from src/service/rooms/timeline/stitcher/test/parser.rs rename to src/stitcher/test/parser.rs index 9a106430..743e0691 100644 --- a/src/service/rooms/timeline/stitcher/test/parser.rs +++ b/src/stitcher/test/parser.rs @@ -2,7 +2,7 @@ use std::collections::HashSet; use indexmap::IndexMap; -use crate::rooms::timeline::stitcher::StitchedItem; +use super::StitchedItem; pub(super) type TestEventId<'id> = &'id str; diff --git a/src/service/rooms/timeline/stitcher/test/testcases/001-receiving_new_events.stitched b/src/stitcher/test/testcases/001-receiving_new_events.stitched similarity index 100% rename from src/service/rooms/timeline/stitcher/test/testcases/001-receiving_new_events.stitched rename to src/stitcher/test/testcases/001-receiving_new_events.stitched diff --git a/src/service/rooms/timeline/stitcher/test/testcases/002-recovering_after_netsplit.stitched b/src/stitcher/test/testcases/002-recovering_after_netsplit.stitched similarity index 100% rename from src/service/rooms/timeline/stitcher/test/testcases/002-recovering_after_netsplit.stitched rename to src/stitcher/test/testcases/002-recovering_after_netsplit.stitched diff --git a/src/service/rooms/timeline/stitcher/test/testcases/zzz-being_before_a_gap_item_beats_being_after_an_existing_item.stitched b/src/stitcher/test/testcases/zzz-being_before_a_gap_item_beats_being_after_an_existing_item.stitched similarity index 100% rename from src/service/rooms/timeline/stitcher/test/testcases/zzz-being_before_a_gap_item_beats_being_after_an_existing_item.stitched rename to src/stitcher/test/testcases/zzz-being_before_a_gap_item_beats_being_after_an_existing_item.stitched diff --git a/src/service/rooms/timeline/stitcher/test/testcases/zzz-being_before_a_gap_item_beats_being_after_an_existing_item_multiple.stitched b/src/stitcher/test/testcases/zzz-being_before_a_gap_item_beats_being_after_an_existing_item_multiple.stitched similarity index 100% rename from src/service/rooms/timeline/stitcher/test/testcases/zzz-being_before_a_gap_item_beats_being_after_an_existing_item_multiple.stitched rename to src/stitcher/test/testcases/zzz-being_before_a_gap_item_beats_being_after_an_existing_item_multiple.stitched diff --git a/src/service/rooms/timeline/stitcher/test/testcases/zzz-chains_are_reordered_using_prev_events.stitched b/src/stitcher/test/testcases/zzz-chains_are_reordered_using_prev_events.stitched similarity index 100% rename from src/service/rooms/timeline/stitcher/test/testcases/zzz-chains_are_reordered_using_prev_events.stitched rename to src/stitcher/test/testcases/zzz-chains_are_reordered_using_prev_events.stitched diff --git a/src/service/rooms/timeline/stitcher/test/testcases/zzz-empty_then_simple_chain.stitched b/src/stitcher/test/testcases/zzz-empty_then_simple_chain.stitched similarity index 100% rename from src/service/rooms/timeline/stitcher/test/testcases/zzz-empty_then_simple_chain.stitched rename to src/stitcher/test/testcases/zzz-empty_then_simple_chain.stitched diff --git a/src/service/rooms/timeline/stitcher/test/testcases/zzz-empty_then_two_chains.stitched b/src/stitcher/test/testcases/zzz-empty_then_two_chains.stitched similarity index 100% rename from src/service/rooms/timeline/stitcher/test/testcases/zzz-empty_then_two_chains.stitched rename to src/stitcher/test/testcases/zzz-empty_then_two_chains.stitched diff --git a/src/service/rooms/timeline/stitcher/test/testcases/zzz-empty_then_two_chains_interleaved.stitched b/src/stitcher/test/testcases/zzz-empty_then_two_chains_interleaved.stitched similarity index 100% rename from src/service/rooms/timeline/stitcher/test/testcases/zzz-empty_then_two_chains_interleaved.stitched rename to src/stitcher/test/testcases/zzz-empty_then_two_chains_interleaved.stitched diff --git a/src/service/rooms/timeline/stitcher/test/testcases/zzz-filling_in_a_gap_with_a_batch_containing_gaps.stitched b/src/stitcher/test/testcases/zzz-filling_in_a_gap_with_a_batch_containing_gaps.stitched similarity index 100% rename from src/service/rooms/timeline/stitcher/test/testcases/zzz-filling_in_a_gap_with_a_batch_containing_gaps.stitched rename to src/stitcher/test/testcases/zzz-filling_in_a_gap_with_a_batch_containing_gaps.stitched diff --git a/src/service/rooms/timeline/stitcher/test/testcases/zzz-gaps_appear_before_events_referring_to_them.stitched b/src/stitcher/test/testcases/zzz-gaps_appear_before_events_referring_to_them.stitched similarity index 100% rename from src/service/rooms/timeline/stitcher/test/testcases/zzz-gaps_appear_before_events_referring_to_them.stitched rename to src/stitcher/test/testcases/zzz-gaps_appear_before_events_referring_to_them.stitched diff --git a/src/service/rooms/timeline/stitcher/test/testcases/zzz-gaps_appear_before_events_referring_to_them_received_order.stitched b/src/stitcher/test/testcases/zzz-gaps_appear_before_events_referring_to_them_received_order.stitched similarity index 100% rename from src/service/rooms/timeline/stitcher/test/testcases/zzz-gaps_appear_before_events_referring_to_them_received_order.stitched rename to src/stitcher/test/testcases/zzz-gaps_appear_before_events_referring_to_them_received_order.stitched diff --git a/src/service/rooms/timeline/stitcher/test/testcases/zzz-if_prev_events_determine_order_they_override_received.stitched b/src/stitcher/test/testcases/zzz-if_prev_events_determine_order_they_override_received.stitched similarity index 100% rename from src/service/rooms/timeline/stitcher/test/testcases/zzz-if_prev_events_determine_order_they_override_received.stitched rename to src/stitcher/test/testcases/zzz-if_prev_events_determine_order_they_override_received.stitched diff --git a/src/service/rooms/timeline/stitcher/test/testcases/zzz-insert_into_first_of_several_gaps.stitched b/src/stitcher/test/testcases/zzz-insert_into_first_of_several_gaps.stitched similarity index 100% rename from src/service/rooms/timeline/stitcher/test/testcases/zzz-insert_into_first_of_several_gaps.stitched rename to src/stitcher/test/testcases/zzz-insert_into_first_of_several_gaps.stitched diff --git a/src/service/rooms/timeline/stitcher/test/testcases/zzz-insert_into_last_of_several_gaps.stitched b/src/stitcher/test/testcases/zzz-insert_into_last_of_several_gaps.stitched similarity index 100% rename from src/service/rooms/timeline/stitcher/test/testcases/zzz-insert_into_last_of_several_gaps.stitched rename to src/stitcher/test/testcases/zzz-insert_into_last_of_several_gaps.stitched diff --git a/src/service/rooms/timeline/stitcher/test/testcases/zzz-insert_into_middle_of_several_gaps.stitched b/src/stitcher/test/testcases/zzz-insert_into_middle_of_several_gaps.stitched similarity index 100% rename from src/service/rooms/timeline/stitcher/test/testcases/zzz-insert_into_middle_of_several_gaps.stitched rename to src/stitcher/test/testcases/zzz-insert_into_middle_of_several_gaps.stitched diff --git a/src/service/rooms/timeline/stitcher/test/testcases/zzz-linked_events_are_split_across_gaps.stitched b/src/stitcher/test/testcases/zzz-linked_events_are_split_across_gaps.stitched similarity index 100% rename from src/service/rooms/timeline/stitcher/test/testcases/zzz-linked_events_are_split_across_gaps.stitched rename to src/stitcher/test/testcases/zzz-linked_events_are_split_across_gaps.stitched diff --git a/src/service/rooms/timeline/stitcher/test/testcases/zzz-linked_events_in_a_diamond_are_split_across_gaps.stitched b/src/stitcher/test/testcases/zzz-linked_events_in_a_diamond_are_split_across_gaps.stitched similarity index 100% rename from src/service/rooms/timeline/stitcher/test/testcases/zzz-linked_events_in_a_diamond_are_split_across_gaps.stitched rename to src/stitcher/test/testcases/zzz-linked_events_in_a_diamond_are_split_across_gaps.stitched diff --git a/src/service/rooms/timeline/stitcher/test/testcases/zzz-middle_of_batch_matches_gap.stitched b/src/stitcher/test/testcases/zzz-middle_of_batch_matches_gap.stitched similarity index 100% rename from src/service/rooms/timeline/stitcher/test/testcases/zzz-middle_of_batch_matches_gap.stitched rename to src/stitcher/test/testcases/zzz-middle_of_batch_matches_gap.stitched diff --git a/src/service/rooms/timeline/stitcher/test/testcases/zzz-middle_of_batch_matches_gap_and_end_of_batch_matches_end.stitched b/src/stitcher/test/testcases/zzz-middle_of_batch_matches_gap_and_end_of_batch_matches_end.stitched similarity index 100% rename from src/service/rooms/timeline/stitcher/test/testcases/zzz-middle_of_batch_matches_gap_and_end_of_batch_matches_end.stitched rename to src/stitcher/test/testcases/zzz-middle_of_batch_matches_gap_and_end_of_batch_matches_end.stitched diff --git a/src/service/rooms/timeline/stitcher/test/testcases/zzz-multiple_events_referring_to_the_same_missing_event.stitched b/src/stitcher/test/testcases/zzz-multiple_events_referring_to_the_same_missing_event.stitched similarity index 100% rename from src/service/rooms/timeline/stitcher/test/testcases/zzz-multiple_events_referring_to_the_same_missing_event.stitched rename to src/stitcher/test/testcases/zzz-multiple_events_referring_to_the_same_missing_event.stitched diff --git a/src/service/rooms/timeline/stitcher/test/testcases/zzz-multiple_events_referring_to_the_same_missing_event_first_has_more.stitched b/src/stitcher/test/testcases/zzz-multiple_events_referring_to_the_same_missing_event_first_has_more.stitched similarity index 100% rename from src/service/rooms/timeline/stitcher/test/testcases/zzz-multiple_events_referring_to_the_same_missing_event_first_has_more.stitched rename to src/stitcher/test/testcases/zzz-multiple_events_referring_to_the_same_missing_event_first_has_more.stitched diff --git a/src/service/rooms/timeline/stitcher/test/testcases/zzz-multiple_events_referring_to_the_same_missing_event_with_more.stitched b/src/stitcher/test/testcases/zzz-multiple_events_referring_to_the_same_missing_event_with_more.stitched similarity index 100% rename from src/service/rooms/timeline/stitcher/test/testcases/zzz-multiple_events_referring_to_the_same_missing_event_with_more.stitched rename to src/stitcher/test/testcases/zzz-multiple_events_referring_to_the_same_missing_event_with_more.stitched diff --git a/src/service/rooms/timeline/stitcher/test/testcases/zzz-multiple_missing_prev_events_turn_into_a_single_gap.stitched b/src/stitcher/test/testcases/zzz-multiple_missing_prev_events_turn_into_a_single_gap.stitched similarity index 100% rename from src/service/rooms/timeline/stitcher/test/testcases/zzz-multiple_missing_prev_events_turn_into_a_single_gap.stitched rename to src/stitcher/test/testcases/zzz-multiple_missing_prev_events_turn_into_a_single_gap.stitched diff --git a/src/service/rooms/timeline/stitcher/test/testcases/zzz-partially_filling_a_gap_leaves_it_before_new_nodes.stitched b/src/stitcher/test/testcases/zzz-partially_filling_a_gap_leaves_it_before_new_nodes.stitched similarity index 100% rename from src/service/rooms/timeline/stitcher/test/testcases/zzz-partially_filling_a_gap_leaves_it_before_new_nodes.stitched rename to src/stitcher/test/testcases/zzz-partially_filling_a_gap_leaves_it_before_new_nodes.stitched diff --git a/src/service/rooms/timeline/stitcher/test/testcases/zzz-partially_filling_a_gap_with_two_events.stitched b/src/stitcher/test/testcases/zzz-partially_filling_a_gap_with_two_events.stitched similarity index 100% rename from src/service/rooms/timeline/stitcher/test/testcases/zzz-partially_filling_a_gap_with_two_events.stitched rename to src/stitcher/test/testcases/zzz-partially_filling_a_gap_with_two_events.stitched diff --git a/src/service/rooms/timeline/stitcher/test/testcases/zzz-received_order_wins_within_a_subgroup_if_no_prev_event_chain.stitched b/src/stitcher/test/testcases/zzz-received_order_wins_within_a_subgroup_if_no_prev_event_chain.stitched similarity index 100% rename from src/service/rooms/timeline/stitcher/test/testcases/zzz-received_order_wins_within_a_subgroup_if_no_prev_event_chain.stitched rename to src/stitcher/test/testcases/zzz-received_order_wins_within_a_subgroup_if_no_prev_event_chain.stitched diff --git a/src/service/rooms/timeline/stitcher/test/testcases/zzz-subgroups_are_processed_in_first_received_order.stitched b/src/stitcher/test/testcases/zzz-subgroups_are_processed_in_first_received_order.stitched similarity index 100% rename from src/service/rooms/timeline/stitcher/test/testcases/zzz-subgroups_are_processed_in_first_received_order.stitched rename to src/stitcher/test/testcases/zzz-subgroups_are_processed_in_first_received_order.stitched