diff --git a/Cargo.lock b/Cargo.lock index d89bfe8f..e256c5ba 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1215,6 +1215,7 @@ dependencies = [ "log", "loole", "lru-cache", + "peg", "rand 0.8.5", "recaptcha-verify", "regex", @@ -3658,6 +3659,33 @@ dependencies = [ "syn", ] +[[package]] +name = "peg" +version = "0.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9928cfca101b36ec5163e70049ee5368a8a1c3c6efc9ca9c5f9cc2f816152477" +dependencies = [ + "peg-macros", + "peg-runtime", +] + +[[package]] +name = "peg-macros" +version = "0.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6298ab04c202fa5b5d52ba03269fb7b74550b150323038878fe6c372d8280f71" +dependencies = [ + "peg-runtime", + "proc-macro2", + "quote", +] + +[[package]] +name = "peg-runtime" +version = "0.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "132dca9b868d927b35b5dd728167b2dee150eb1ad686008fc71ccb298b776fca" + [[package]] name = "percent-encoding" version = "2.3.2" diff --git a/src/service/Cargo.toml b/src/service/Cargo.toml index f8fc932f..d24b8a22 100644 --- a/src/service/Cargo.toml +++ b/src/service/Cargo.toml @@ -125,3 +125,6 @@ sd-notify.optional = true [lints] workspace = true + +[dev-dependencies] +peg = "0.8.5" diff --git a/src/service/rooms/timeline/stitcher/algorithm.rs b/src/service/rooms/timeline/stitcher/algorithm.rs index c9ef73bc..e4db7e3b 100644 --- a/src/service/rooms/timeline/stitcher/algorithm.rs +++ b/src/service/rooms/timeline/stitcher/algorithm.rs @@ -4,11 +4,11 @@ use std::{ }; use itertools::Itertools; -use ruma::{EventId, OwnedEventId}; use super::{Batch, Gap, OrderKey, StitchedItem, StitcherBackend}; /// Updates to a gap in the stitched order. +#[derive(Debug)] pub(super) struct GapUpdate<'id, K: OrderKey> { /// The opaque key of the gap to update. pub key: K, @@ -21,6 +21,7 @@ pub(super) struct GapUpdate<'id, K: OrderKey> { } /// Updates to the stitched order. +#[derive(Debug)] pub(super) struct OrderUpdates<'id, K: OrderKey> { /// Updates to individual gaps. The items inserted by these updates _should /// not_ be synchronized to clients. @@ -35,14 +36,12 @@ pub(super) struct Stitcher<'backend, B: StitcherBackend> { } impl Stitcher<'_, B> { - pub(super) fn new<'backend>(backend: &'backend B) -> Stitcher<'backend, B> { - Stitcher { backend } - } + pub(super) fn new(backend: &B) -> Stitcher<'_, B> { Stitcher { backend } } pub(super) fn stitch<'id>(&self, batch: Batch<'id>) -> OrderUpdates<'id, B::Key> { let mut gap_updates = Vec::new(); - let mut remaining_events: BTreeSet<&EventId> = batch.events().collect(); + let mut remaining_events: BTreeSet<_> = batch.events().collect(); // 1: Find existing gaps which include IDs of events in `batch` let matching_gaps = self.backend.find_matching_gaps(batch.events()); @@ -53,7 +52,7 @@ impl Stitcher<'_, B> { let matching_events = remaining_events.iter().filter(|id| gap.contains(**id)); // 3. Create the to-insert list from the predecessor sets of each matching event - let events_to_insert: Vec<&'id EventId> = matching_events + let events_to_insert: Vec<_> = matching_events .filter_map(|event| batch.predecessors(event)) .flat_map(|predecessors| predecessors.predecessor_set.iter()) .filter(|event| remaining_events.contains(*event)) @@ -84,7 +83,7 @@ impl Stitcher<'_, B> { fn sort_events_and_create_gaps<'id>( &self, batch: &Batch<'id>, - events_to_insert: impl IntoIterator, + events_to_insert: impl IntoIterator, ) -> Vec> { // 5. Sort the to-insert list with DAG;received order let events_to_insert = events_to_insert @@ -96,8 +95,8 @@ impl Stitcher<'_, B> { events_to_insert.capacity() + events_to_insert.capacity().div_euclid(2), ); - for event in events_to_insert.into_iter() { - let missing_prev_events: HashSet = batch + for event in events_to_insert { + let missing_prev_events: HashSet = batch .predecessors(event) .expect("events in to_insert should be in batch") .prev_events @@ -105,14 +104,14 @@ impl Stitcher<'_, B> { .filter(|prev_event| { !(batch.contains(prev_event) || self.backend.event_exists(prev_event)) }) - .map(|id| OwnedEventId::from(*id)) + .map(|id| String::from(*id)) .collect(); if !missing_prev_events.is_empty() { items.push(StitchedItem::Gap(missing_prev_events)); } - items.push(StitchedItem::Event(event)) + items.push(StitchedItem::Event(event)); } items @@ -124,7 +123,7 @@ impl Stitcher<'_, B> { /// otherwise they are sorted by which comes first in the batch. fn compare_by_dag_received<'id>( batch: &Batch<'id>, - ) -> impl FnMut(&&'id EventId, &&'id EventId) -> Ordering { + ) -> impl FnMut(&&'id str, &&'id str) -> Ordering { |a, b| { if batch .predecessors(a) @@ -145,7 +144,7 @@ impl Stitcher<'_, B> { } } - panic!("neither {} nor {} in batch", a, b); + panic!("neither {a} nor {b} in batch"); } } } diff --git a/src/service/rooms/timeline/stitcher/mod.rs b/src/service/rooms/timeline/stitcher/mod.rs index 66b55989..9b9e43cf 100644 --- a/src/service/rooms/timeline/stitcher/mod.rs +++ b/src/service/rooms/timeline/stitcher/mod.rs @@ -1,14 +1,15 @@ use std::collections::{BTreeMap, HashSet}; -use ruma::{EventId, OwnedEventId}; - pub(super) mod algorithm; +#[cfg(test)] +mod test; /// A gap in the stitched order. -pub(super) type Gap = HashSet; +pub(super) type Gap = HashSet; +#[derive(Debug)] pub(super) enum StitchedItem<'id> { - Event(&'id EventId), + Event(&'id str), Gap(Gap), } @@ -16,40 +17,44 @@ pub(super) enum StitchedItem<'id> { /// order. pub(super) trait OrderKey: Eq + Clone {} +impl OrderKey for T {} + pub(super) trait StitcherBackend { type Key: OrderKey; + /// Returns all gaps containing an event listed in `events`. fn find_matching_gaps<'a>( &'a self, - events: impl Iterator, + events: impl Iterator, ) -> impl Iterator; - fn event_exists<'a>(&'a self, event: &'a EventId) -> bool; + /// Returns whether an event exists in the stitched order. + fn event_exists<'a>(&'a self, event: &'a str) -> bool; } /// An ordered map from an event ID to its `prev_events`. -pub(super) type EventEdges<'id> = BTreeMap<&'id EventId, HashSet<&'id EventId>>; +pub(super) type EventEdges<'id> = BTreeMap<&'id str, HashSet<&'id str>>; /// Information about the `prev_events` of an event. /// This struct does not store the ID of the event itself. struct EventPredecessors<'id> { /// The `prev_events` of the event. - pub prev_events: HashSet<&'id EventId>, + pub prev_events: HashSet<&'id str>, /// The predecessor set of the event. This is a superset of /// [`EventPredecessors::prev_events`]. See [`Batch::find_predecessor_set`] /// for details. - pub predecessor_set: HashSet<&'id EventId>, + pub predecessor_set: HashSet<&'id str>, } pub(super) struct Batch<'id> { - events: BTreeMap<&'id EventId, EventPredecessors<'id>>, + events: BTreeMap<&'id str, EventPredecessors<'id>>, } impl<'id> Batch<'id> { - pub(super) fn from_edges<'a>(edges: EventEdges<'a>) -> Batch<'a> { + pub(super) fn from_edges(edges: EventEdges<'_>) -> Batch<'_> { let mut events = BTreeMap::new(); - for (event, prev_events) in edges.iter() { + for (event, prev_events) in &edges { let predecessor_set = Self::find_predecessor_set(event, &edges); events.insert(*event, EventPredecessors { @@ -66,10 +71,7 @@ impl<'id> Batch<'id> { /// rooted at `event` containing _only_ events which are included in /// `edges`. It is represented as a set and not a proper tree structure for /// efficiency. - fn find_predecessor_set<'a>( - event: &'a EventId, - edges: &EventEdges<'a>, - ) -> HashSet<&'a EventId> { + fn find_predecessor_set<'a>(event: &'a str, edges: &EventEdges<'a>) -> HashSet<&'a str> { // The predecessor set which we are building. let mut predecessor_set = HashSet::new(); @@ -100,11 +102,11 @@ impl<'id> Batch<'id> { predecessor_set } - fn events(&self) -> impl Iterator { self.events.keys().copied() } + fn events(&self) -> impl Iterator { self.events.keys().copied() } - fn contains(&self, event: &'id EventId) -> bool { self.events.contains_key(event) } + fn contains(&self, event: &'id str) -> bool { self.events.contains_key(event) } - fn predecessors(&self, event: &EventId) -> Option<&EventPredecessors<'id>> { + fn predecessors(&self, event: &str) -> Option<&EventPredecessors<'id>> { self.events.get(event) } } diff --git a/src/service/rooms/timeline/stitcher/test/mod.rs b/src/service/rooms/timeline/stitcher/test/mod.rs new file mode 100644 index 00000000..47c5de32 --- /dev/null +++ b/src/service/rooms/timeline/stitcher/test/mod.rs @@ -0,0 +1,170 @@ +use std::sync::atomic::{AtomicU64, Ordering}; + +use itertools::Itertools; + +use super::{algorithm::*, *}; +use crate::rooms::timeline::stitcher::algorithm::Stitcher; + +mod parser; + +#[derive(Default)] +struct TestStitcherBackend<'id> { + items: Vec<(u64, StitchedItem<'id>)>, + counter: AtomicU64, +} + +impl<'id> TestStitcherBackend<'id> { + fn next_id(&self) -> u64 { self.counter.fetch_add(1, Ordering::Relaxed) } + + fn extend(&mut self, results: OrderUpdates<'id, ::Key>) { + for update in results.gap_updates { + let Some(gap_index) = self.items.iter().position(|(key, _)| *key == update.key) + else { + panic!("bad update key {}", update.key); + }; + + let insertion_index; + if update.gap.is_empty() { + self.items.remove(gap_index); + insertion_index = gap_index; + } else { + self.items[gap_index] = (self.next_id(), StitchedItem::Gap(update.gap)); + insertion_index = gap_index + 1; + } + + let to_insert: Vec<_> = update + .inserted_items + .into_iter() + .map(|item| (self.next_id(), item)) + .collect(); + self.items + .splice(insertion_index..insertion_index, to_insert.into_iter()) + .for_each(drop); + } + + let new_items: Vec<_> = results + .new_items + .into_iter() + .map(|item| (self.next_id(), item)) + .collect(); + self.items.extend(new_items); + } + + fn iter(&self) -> impl Iterator> { + self.items.iter().map(|(_, item)| item) + } +} + +impl StitcherBackend for TestStitcherBackend<'_> { + type Key = u64; + + fn find_matching_gaps<'a>( + &'a self, + events: impl Iterator, + ) -> impl Iterator { + // nobody cares about test suite performance right + let mut gaps = vec![]; + + for event in events { + for (key, item) in &self.items { + if let StitchedItem::Gap(gap) = item + && gap.contains(event) + { + gaps.push((*key, gap.clone())); + } + } + } + + gaps.into_iter() + } + + fn event_exists<'a>(&'a self, event: &'a str) -> bool { + self.items + .iter() + .any(|item| matches!(item.1, StitchedItem::Event(id) if event == id)) + } +} + +fn run_testcase(testcase: parser::TestCase<'_>) { + let mut backend = TestStitcherBackend::default(); + + for phase in testcase { + let stitcher = Stitcher::new(&backend); + let batch = Batch::from_edges(phase.batch); + let updates = stitcher.stitch(batch); + + println!("updates to make: {:?}", updates); + + for (expected, actual) in phase + .order + .new_items + .iter() + .zip_eq(updates.new_items.iter()) + { + assert_eq!( + expected, actual, + "bad new item, expected {:?} but got {:?}", + expected, actual + ); + } + + backend.extend(updates); + + println!("ordering: {:?}", backend.items); + + for (expected, actual) in phase.order.iter().zip_eq(backend.iter()) { + assert_eq!( + expected, actual, + "bad item in order, expected {:?} but got {:?}", + expected, actual + ); + } + + // TODO gap notification + } +} + +macro_rules! testcase { + ($index:literal : $id:ident) => { + #[test] + fn $id() { + let testcase = parser::parse(include_str!(concat!( + "./testcases/", + $index, + "-", + stringify!($id), + ".stitched" + ))); + + run_testcase(testcase); + } + }; +} + +testcase!("001": receiving_new_events); +testcase!("002": recovering_after_netsplit); +testcase!("zzz": being_before_a_gap_item_beats_being_after_an_existing_item_multiple); +testcase!("zzz": being_before_a_gap_item_beats_being_after_an_existing_item); +testcase!("zzz": chains_are_reordered_using_prev_events); +testcase!("zzz": empty_then_simple_chain); +testcase!("zzz": empty_then_two_chains_interleaved); +testcase!("zzz": empty_then_two_chains); +testcase!("zzz": filling_in_a_gap_with_a_batch_containing_gaps); +testcase!("zzz": gaps_appear_before_events_referring_to_them_received_order); +testcase!("zzz": gaps_appear_before_events_referring_to_them); +testcase!("zzz": if_prev_events_determine_order_they_override_received); +testcase!("zzz": insert_into_first_of_several_gaps); +testcase!("zzz": insert_into_last_of_several_gaps); +testcase!("zzz": insert_into_middle_of_several_gaps); +testcase!("zzz": linked_events_are_split_across_gaps); +testcase!("zzz": linked_events_in_a_diamond_are_split_across_gaps); +testcase!("zzz": middle_of_batch_matches_gap_and_end_of_batch_matches_end); +testcase!("zzz": middle_of_batch_matches_gap); +testcase!("zzz": multiple_events_referring_to_the_same_missing_event_first_has_more); +testcase!("zzz": multiple_events_referring_to_the_same_missing_event); +testcase!("zzz": multiple_events_referring_to_the_same_missing_event_with_more); +testcase!("zzz": multiple_missing_prev_events_turn_into_a_single_gap); +testcase!("zzz": partially_filling_a_gap_leaves_it_before_new_nodes); +testcase!("zzz": partially_filling_a_gap_with_two_events); +testcase!("zzz": received_order_wins_within_a_subgroup_if_no_prev_event_chain); +testcase!("zzz": subgroups_are_processed_in_first_received_order); diff --git a/src/service/rooms/timeline/stitcher/test/parser.rs b/src/service/rooms/timeline/stitcher/test/parser.rs new file mode 100644 index 00000000..bd914c6f --- /dev/null +++ b/src/service/rooms/timeline/stitcher/test/parser.rs @@ -0,0 +1,138 @@ +use std::collections::{BTreeMap, HashSet}; + +use crate::rooms::timeline::stitcher::{StitchedItem, test}; + +pub(super) type TestEventId<'id> = &'id str; + +pub(super) type TestGap<'id> = HashSet>; + +#[derive(Debug)] +pub(super) enum TestStitchedItem<'id> { + Event(TestEventId<'id>), + Gap(TestGap<'id>), +} + +impl PartialEq> for TestStitchedItem<'_> { + fn eq(&self, other: &StitchedItem<'_>) -> bool { + match (self, other) { + | (TestStitchedItem::Event(lhs), StitchedItem::Event(rhs)) => lhs == rhs, + | (TestStitchedItem::Gap(lhs), StitchedItem::Gap(rhs)) => + lhs.iter().all(|id| rhs.contains(*id)), + | _ => false, + } + } +} + +pub(super) type TestCase<'id> = Vec>; + +pub(super) struct Phase<'id> { + pub batch: Batch<'id>, + pub order: Order<'id>, + pub updated_gaps: Option>>, +} + +pub(super) type Batch<'id> = BTreeMap, HashSet>>; + +pub(super) struct Order<'id> { + pub inserted_items: Vec>, + pub new_items: Vec>, +} + +impl<'id> Order<'id> { + pub(super) fn iter(&self) -> impl Iterator> { + self.inserted_items.iter().chain(self.new_items.iter()) + } +} + +peg::parser! { + grammar testcase() for str { + /// Parse whitespace. + rule _ -> () = quiet! { $([' '])* {} } + + /// Parse empty lines and comments. + rule newline() -> () = quiet! { (("#" [^'\n']*)? "\n")+ {} } + + /// Parse an "event ID" in a test case, which may only consist of ASCII letters and numbers. + rule event_id() -> TestEventId<'input> + = quiet! { id:$([char if char.is_ascii_alphanumeric()]+) { id } } + / expected!("event id") + + /// Parse a gap in the order section. + rule gap() -> TestGap<'input> + = "-" events:event_id() ++ "," { events.into_iter().collect() } + + /// Parse either an event id or a gap. + rule stitched_item() -> TestStitchedItem<'input> = + id:event_id() { TestStitchedItem::Event(id) } + / gap:gap() { TestStitchedItem::Gap(gap) } + + /// Parse an event line in the batch section, mapping an event name to zero or one prev events. + /// The prev events are merged together by [`batch()`]. + rule batch_event() -> (TestEventId<'input>, Option>) + = id:event_id() prev:(_ "-->" _ prev:event_id() { prev })? { (id, prev) } + + /// Parse the batch section of a phase. + rule batch() -> Batch<'input> + = events:batch_event() ++ newline() { + /* + Repeated event lines need to be merged together. For example, + + A --> B + A --> C + + represents a _single_ event `A` with two prev events, `B` and `C`. + */ + events.into_iter() + .fold(BTreeMap::new(), |mut batch: Batch<'_>, (id, prev_event)| { + // Find the prev events set of this event in the batch. + // If it doesn't exist, make a new empty one. + let mut prev_events = batch.entry(id).or_default(); + // If this event line defines a prev event to add, insert it into the set. + if let Some(prev_event) = prev_event { + prev_events.insert(prev_event); + } + + batch + }) + } + + rule order() -> Order<'input> = + items:(item:stitched_item() new:"*"? { (item, new.is_some()) }) ** newline() + { + let (mut inserted_items, mut new_items) = (vec![], vec![]); + + for (item, new) in items { + if new { + new_items.push(item); + } else { + inserted_items.push(item); + } + } + + Order { + inserted_items, + new_items, + } + } + + rule updated_gaps() -> HashSet> = + events:event_id() ++ newline() { events.into_iter().collect() } + + rule phase() -> Phase<'input> = + "=== when we receive these events ===" + newline() batch:batch() + newline() "=== then we arrange into this order ===" + newline() order:order() + updated_gaps:( + newline() "=== and we notify about these gaps ===" + newline() updated_gaps:updated_gaps() { updated_gaps } + )? + { Phase { batch, order, updated_gaps } } + + pub rule testcase() -> TestCase<'input> = phase() ++ newline() + } +} + +pub(super) fn parse<'input>(input: &'input str) -> TestCase<'input> { + testcase::testcase(input.trim_ascii_end()).expect("parse error") +} diff --git a/src/service/rooms/timeline/stitcher/test/testcases/001-receiving_new_events.stitched b/src/service/rooms/timeline/stitcher/test/testcases/001-receiving_new_events.stitched new file mode 100644 index 00000000..ce518340 --- /dev/null +++ b/src/service/rooms/timeline/stitcher/test/testcases/001-receiving_new_events.stitched @@ -0,0 +1,22 @@ +=== when we receive these events === +A +B --> A +C --> B +=== then we arrange into this order === +# Given the server has some existing events in this order: +A* +B* +C* + +=== when we receive these events === +# When it receives new ones: +D --> C +E --> D + +=== then we arrange into this order === +# Then it simply appends them at the end of the order: +A +B +C +D* +E* diff --git a/src/service/rooms/timeline/stitcher/test/testcases/002-recovering_after_netsplit.stitched b/src/service/rooms/timeline/stitcher/test/testcases/002-recovering_after_netsplit.stitched new file mode 100644 index 00000000..9dcc67e2 --- /dev/null +++ b/src/service/rooms/timeline/stitcher/test/testcases/002-recovering_after_netsplit.stitched @@ -0,0 +1,46 @@ +=== when we receive these events === +A1 +A2 --> A1 +A3 --> A2 +=== then we arrange into this order === +# Given the server has some existing events in this order: +A1* +A2* +A3* + +=== when we receive these events === +# And after a netsplit the server receives some unrelated events, which refer to +# some unknown event, because the server didn't receive all of them: +B7 --> B6 +B8 --> B7 +B9 --> B8 + +=== then we arrange into this order === +# Then these events are new, and we add a gap to show something is missing: +A1 +A2 +A3 +-B6* +B7* +B8* +B9* +=== when we receive these events === +# Then if we backfill and receive more of those events later: +B4 --> B3 +B5 --> B4 +B6 --> B5 +=== then we arrange into this order === +# They are slotted into the gap, and a new gap is created to represent the +# still-missing events: +A1 +A2 +A3 +-B3 +B4 +B5 +B6 +B7 +B8 +B9 +=== and we notify about these gaps === +B6 diff --git a/src/service/rooms/timeline/stitcher/test/testcases/zzz-being_before_a_gap_item_beats_being_after_an_existing_item.stitched b/src/service/rooms/timeline/stitcher/test/testcases/zzz-being_before_a_gap_item_beats_being_after_an_existing_item.stitched new file mode 100644 index 00000000..7853b25b --- /dev/null +++ b/src/service/rooms/timeline/stitcher/test/testcases/zzz-being_before_a_gap_item_beats_being_after_an_existing_item.stitched @@ -0,0 +1,30 @@ +=== when we receive these events === +D --> C +=== then we arrange into this order === +# We may see situations that are ambiguous about whether an event is new or +# belongs in a gap, because it is a predecessor of a gap event and also has a +# new event as its predecessor. This a rare case where either outcome could be +# valid. If the initial order is this: +-C* +D* +=== when we receive these events === +# And then we receive B +B --> A +=== then we arrange into this order === +# Which is new because it's unrelated to everything else +-C +D +-A* +B* +=== when we receive these events === +# And later it turns out that C refers back to B +C --> B +=== then we arrange into this order === +# Then we place C into the early gap even though it is after B, so arguably +# should be the newest +C +D +-A +B +=== and we notify about these gaps === +C diff --git a/src/service/rooms/timeline/stitcher/test/testcases/zzz-being_before_a_gap_item_beats_being_after_an_existing_item_multiple.stitched b/src/service/rooms/timeline/stitcher/test/testcases/zzz-being_before_a_gap_item_beats_being_after_an_existing_item_multiple.stitched new file mode 100644 index 00000000..37dfbd59 --- /dev/null +++ b/src/service/rooms/timeline/stitcher/test/testcases/zzz-being_before_a_gap_item_beats_being_after_an_existing_item_multiple.stitched @@ -0,0 +1,28 @@ +=== when we receive these events === +# An ambiguous situation can occur when we have multiple gaps that both might +# accepts an event. This should be relatively rare. +A --> G1 +B --> A +C --> G2 +=== then we arrange into this order === +-G1* +A +B +-G2* +C +=== when we receive these events === +# When we receive F, which is a predecessor of both G1 and G2 +F +G1 --> F +G2 --> F +=== then we arrange into this order === +# Then F appears in the earlier gap, but arguably it should appear later. +F +G1 +A +B +G2 +C +=== and we notify about these gaps === +G1 +G2 diff --git a/src/service/rooms/timeline/stitcher/test/testcases/zzz-chains_are_reordered_using_prev_events.stitched b/src/service/rooms/timeline/stitcher/test/testcases/zzz-chains_are_reordered_using_prev_events.stitched new file mode 100644 index 00000000..7b2f8f3c --- /dev/null +++ b/src/service/rooms/timeline/stitcher/test/testcases/zzz-chains_are_reordered_using_prev_events.stitched @@ -0,0 +1,10 @@ +=== when we receive these events === +# Even though we see C first, it is re-ordered because we must obey prev_events +# so A comes first. +C --> A +A +B --> A +=== then we arrange into this order === +A* +C* +B* diff --git a/src/service/rooms/timeline/stitcher/test/testcases/zzz-empty_then_simple_chain.stitched b/src/service/rooms/timeline/stitcher/test/testcases/zzz-empty_then_simple_chain.stitched new file mode 100644 index 00000000..4dea04ad --- /dev/null +++ b/src/service/rooms/timeline/stitcher/test/testcases/zzz-empty_then_simple_chain.stitched @@ -0,0 +1,8 @@ +=== when we receive these events === +A +B --> A +C --> B +=== then we arrange into this order === +A* +B* +C* diff --git a/src/service/rooms/timeline/stitcher/test/testcases/zzz-empty_then_two_chains.stitched b/src/service/rooms/timeline/stitcher/test/testcases/zzz-empty_then_two_chains.stitched new file mode 100644 index 00000000..e31f668a --- /dev/null +++ b/src/service/rooms/timeline/stitcher/test/testcases/zzz-empty_then_two_chains.stitched @@ -0,0 +1,18 @@ +=== when we receive these events === +# A chain ABC +A +B --> A +C --> B +# And a separate chain XYZ +X --> W +Y --> X +Z --> Y +=== then we arrange into this order === +# Should produce them in order with a gap +A* +B* +C* +-W* +X* +Y* +Z* diff --git a/src/service/rooms/timeline/stitcher/test/testcases/zzz-empty_then_two_chains_interleaved.stitched b/src/service/rooms/timeline/stitcher/test/testcases/zzz-empty_then_two_chains_interleaved.stitched new file mode 100644 index 00000000..2065eb90 --- /dev/null +++ b/src/service/rooms/timeline/stitcher/test/testcases/zzz-empty_then_two_chains_interleaved.stitched @@ -0,0 +1,18 @@ +=== when we receive these events === +# Same as empty_then_two_chains except for received order +# A chain ABC, and a separate chain XYZ, but interleaved +A +X --> W +B --> A +Y --> X +C --> B +Z --> Y +=== then we arrange into this order === +# Should produce them in order with a gap +A* +-W* +X* +B* +Y* +C* +Z* diff --git a/src/service/rooms/timeline/stitcher/test/testcases/zzz-filling_in_a_gap_with_a_batch_containing_gaps.stitched b/src/service/rooms/timeline/stitcher/test/testcases/zzz-filling_in_a_gap_with_a_batch_containing_gaps.stitched new file mode 100644 index 00000000..bd5da41e --- /dev/null +++ b/src/service/rooms/timeline/stitcher/test/testcases/zzz-filling_in_a_gap_with_a_batch_containing_gaps.stitched @@ -0,0 +1,33 @@ +=== when we receive these events === +# Given 3 gaps exist +B --> A +D --> C +F --> E +=== then we arrange into this order === +-A* +B* +-C* +D* +-E* +F* + +=== when we receive these events === +# When we fill one with something that also refers to non-existent events +C --> X +C --> Y +G --> C +G --> Z +=== then we arrange into this order === +# Then we fill in the gap (C) and make new gaps too (X+Y and Z) +-A +B +-X,Y +C +D +-E +F +-Z* +G* +=== and we notify about these gaps === +# And we notify about the gap that was updated +C diff --git a/src/service/rooms/timeline/stitcher/test/testcases/zzz-gaps_appear_before_events_referring_to_them.stitched b/src/service/rooms/timeline/stitcher/test/testcases/zzz-gaps_appear_before_events_referring_to_them.stitched new file mode 100644 index 00000000..cf4170c2 --- /dev/null +++ b/src/service/rooms/timeline/stitcher/test/testcases/zzz-gaps_appear_before_events_referring_to_them.stitched @@ -0,0 +1,13 @@ +=== when we receive these events === +# Several events refer to missing events and the events are unrelated +C --> Y +C --> Z +A --> X +B +=== then we arrange into this order === +# The gaps appear immediately before the events referring to them +-Y,Z* +C* +-X* +A* +B* diff --git a/src/service/rooms/timeline/stitcher/test/testcases/zzz-gaps_appear_before_events_referring_to_them_received_order.stitched b/src/service/rooms/timeline/stitcher/test/testcases/zzz-gaps_appear_before_events_referring_to_them_received_order.stitched new file mode 100644 index 00000000..5cd6886f --- /dev/null +++ b/src/service/rooms/timeline/stitcher/test/testcases/zzz-gaps_appear_before_events_referring_to_them_received_order.stitched @@ -0,0 +1,14 @@ +=== when we receive these events === +# Several events refer to missing events and the events are related +C --> Y +C --> Z +C --> B +A --> X +B --> A +=== then we arrange into this order === +# The gaps appear immediately before the events referring to them +-X* +A* +B* +-Y,Z* +C* diff --git a/src/service/rooms/timeline/stitcher/test/testcases/zzz-if_prev_events_determine_order_they_override_received.stitched b/src/service/rooms/timeline/stitcher/test/testcases/zzz-if_prev_events_determine_order_they_override_received.stitched new file mode 100644 index 00000000..b4e28403 --- /dev/null +++ b/src/service/rooms/timeline/stitcher/test/testcases/zzz-if_prev_events_determine_order_they_override_received.stitched @@ -0,0 +1,15 @@ +=== when we receive these events === +# The relationships determine the order here, so they override received order +F --> E +C --> B +D --> C +E --> D +B --> A +A +=== then we arrange into this order === +A* +B* +C* +D* +E* +F* diff --git a/src/service/rooms/timeline/stitcher/test/testcases/zzz-insert_into_first_of_several_gaps.stitched b/src/service/rooms/timeline/stitcher/test/testcases/zzz-insert_into_first_of_several_gaps.stitched new file mode 100644 index 00000000..83655479 --- /dev/null +++ b/src/service/rooms/timeline/stitcher/test/testcases/zzz-insert_into_first_of_several_gaps.stitched @@ -0,0 +1,27 @@ +=== when we receive these events === +# Given 3 gaps exist +B --> A +D --> C +F --> E +=== then we arrange into this order === +-A* +B* +-C* +D* +-E* +F* + +=== when we receive these events === +# When the first of them is filled in +A +=== then we arrange into this order === +# Then we slot it into the gap, not at the end +A +B +-C +D +-E +F +=== and we notify about these gaps === +# And we notify about the gap being filled in +A diff --git a/src/service/rooms/timeline/stitcher/test/testcases/zzz-insert_into_last_of_several_gaps.stitched b/src/service/rooms/timeline/stitcher/test/testcases/zzz-insert_into_last_of_several_gaps.stitched new file mode 100644 index 00000000..95b3e315 --- /dev/null +++ b/src/service/rooms/timeline/stitcher/test/testcases/zzz-insert_into_last_of_several_gaps.stitched @@ -0,0 +1,27 @@ +=== when we receive these events === +# Given 3 gaps exist +B --> A +D --> C +F --> E +=== then we arrange into this order === +-A* +B* +-C* +D* +-E* +F* + +=== when we receive these events === +# When the last gap is filled in +E +=== then we arrange into this order === +# Then we slot it into the gap, not at the end +-A +B +-C +D +E +F +=== and we notify about these gaps === +# And we notify about the gap being filled in +E diff --git a/src/service/rooms/timeline/stitcher/test/testcases/zzz-insert_into_middle_of_several_gaps.stitched b/src/service/rooms/timeline/stitcher/test/testcases/zzz-insert_into_middle_of_several_gaps.stitched new file mode 100644 index 00000000..49deda00 --- /dev/null +++ b/src/service/rooms/timeline/stitcher/test/testcases/zzz-insert_into_middle_of_several_gaps.stitched @@ -0,0 +1,27 @@ +=== when we receive these events === +# Given 3 gaps exist +B --> A +D --> C +F --> E +=== then we arrange into this order === +-A* +B* +-C* +D* +-E* +F* + +=== when we receive these events === +# When a middle one is filled in +C +=== then we arrange into this order === +# Then we slot it into the gap, not at the end +-A +B +C +D +-E +F +=== and we notify about these gaps === +# And we notify about the gap being filled in +C diff --git a/src/service/rooms/timeline/stitcher/test/testcases/zzz-linked_events_are_split_across_gaps.stitched b/src/service/rooms/timeline/stitcher/test/testcases/zzz-linked_events_are_split_across_gaps.stitched new file mode 100644 index 00000000..2510549f --- /dev/null +++ b/src/service/rooms/timeline/stitcher/test/testcases/zzz-linked_events_are_split_across_gaps.stitched @@ -0,0 +1,29 @@ +=== when we receive these events === +# Given a couple of gaps +B --> X2 +D --> X4 +=== then we arrange into this order === +-X2* +B* +-X4* +D* + +=== when we receive these events === +# And linked events that fill those in and are newer +X1 +X2 --> X1 +X3 --> X2 +X4 --> X3 +X5 --> X4 +=== then we arrange into this order === +# Then the gaps are filled and new events appear at the front +X1 +X2 +B +X3 +X4 +D +X5* +=== and we notify about these gaps === +X2 +X4 diff --git a/src/service/rooms/timeline/stitcher/test/testcases/zzz-linked_events_in_a_diamond_are_split_across_gaps.stitched b/src/service/rooms/timeline/stitcher/test/testcases/zzz-linked_events_in_a_diamond_are_split_across_gaps.stitched new file mode 100644 index 00000000..1d9a708e --- /dev/null +++ b/src/service/rooms/timeline/stitcher/test/testcases/zzz-linked_events_in_a_diamond_are_split_across_gaps.stitched @@ -0,0 +1,31 @@ +=== when we receive these events === +# Given a couple of gaps +B --> X2a +D --> X3 +=== then we arrange into this order === +-X2a* +B* +-X3* +D* + +=== when we receive these events === +# When we receive a diamond that touches gaps and some new events +X1 +X2a --> X1 +X2b --> X1 +X3 --> X2a +X3 --> X2b +X4 --> X3 +=== then we arrange into this order === +# Then matching events and direct predecessors fit into the gaps +# and other stuff is new +X1 +X2a +B +X2b +X3 +D +X4* +=== and we notify about these gaps === +X2a +X3 diff --git a/src/service/rooms/timeline/stitcher/test/testcases/zzz-middle_of_batch_matches_gap.stitched b/src/service/rooms/timeline/stitcher/test/testcases/zzz-middle_of_batch_matches_gap.stitched new file mode 100644 index 00000000..8cb75918 --- /dev/null +++ b/src/service/rooms/timeline/stitcher/test/testcases/zzz-middle_of_batch_matches_gap.stitched @@ -0,0 +1,25 @@ +=== when we receive these events === +# Given a gap before all the Bs +B1 --> C2 +B2 --> B1 +=== then we arrange into this order === +-C2* +B1* +B2* + +=== when we receive these events === +# When a batch arrives with a not-last event matching the gap +C1 +C2 --> C1 +C3 --> C2 +=== then we arrange into this order === +# Then we slot the matching events into the gap +# and the later events are new +C1 +C2 +B1 +B2 +C3* +=== and we notify about these gaps === +# And we notify about the gap being filled in +C2 diff --git a/src/service/rooms/timeline/stitcher/test/testcases/zzz-middle_of_batch_matches_gap_and_end_of_batch_matches_end.stitched b/src/service/rooms/timeline/stitcher/test/testcases/zzz-middle_of_batch_matches_gap_and_end_of_batch_matches_end.stitched new file mode 100644 index 00000000..7965cbb3 --- /dev/null +++ b/src/service/rooms/timeline/stitcher/test/testcases/zzz-middle_of_batch_matches_gap_and_end_of_batch_matches_end.stitched @@ -0,0 +1,26 @@ +=== when we receive these events === +# Given a gap before all the Bs +B1 --> C2 +B2 --> B1 +=== then we arrange into this order === +-C2* +B1* +B2* + +=== when we receive these events === +# When a batch arrives with a not-last event matching the gap, and the last +# event linked to a recent event +C1 +C2 --> C1 +C3 --> C2 +C3 --> B2 +=== then we arrange into this order === +# Then we slot the entire batch into the gap +C1 +C2 +B1 +B2 +C3* +=== and we notify about these gaps === +# And we notify about the gap being filled in +C2 diff --git a/src/service/rooms/timeline/stitcher/test/testcases/zzz-multiple_events_referring_to_the_same_missing_event.stitched b/src/service/rooms/timeline/stitcher/test/testcases/zzz-multiple_events_referring_to_the_same_missing_event.stitched new file mode 100644 index 00000000..d4e88335 --- /dev/null +++ b/src/service/rooms/timeline/stitcher/test/testcases/zzz-multiple_events_referring_to_the_same_missing_event.stitched @@ -0,0 +1,26 @@ +=== when we receive these events === +# If multiple events all refer to the same missing event: +A --> X +B --> X +C --> X +=== then we arrange into this order === +# Then we insert gaps before all of them. This avoids the need to search the +# entire existing order whenever we create a new gap. +-X* +A* +-X* +B* +-X* +C* +=== when we receive these events === +# The ambiguity is resolved when the missing event arrives: +X +=== then we arrange into this order === +# We choose the earliest gap, and all the relevant gaps are removed (which does +# mean we need to search the existing order). +X +A +B +C +=== and we notify about these gaps === +X diff --git a/src/service/rooms/timeline/stitcher/test/testcases/zzz-multiple_events_referring_to_the_same_missing_event_first_has_more.stitched b/src/service/rooms/timeline/stitcher/test/testcases/zzz-multiple_events_referring_to_the_same_missing_event_first_has_more.stitched new file mode 100644 index 00000000..abde5a82 --- /dev/null +++ b/src/service/rooms/timeline/stitcher/test/testcases/zzz-multiple_events_referring_to_the_same_missing_event_first_has_more.stitched @@ -0,0 +1,29 @@ +=== when we receive these events === +# Several events refer to the same missing event, but the first refers to +# others too +A --> X +A --> Y +A --> Z +B --> X +C --> X +=== then we arrange into this order === +# We end up with multiple gaps +-X,Y,Z* +A* +-X* +B* +-X* +C* + +=== when we receive these events === +# When we receive the missing item +X +=== then we arrange into this order === +# It goes into the earliest slot, and the non-empty gap remains +-Y,Z +X +A +B +C +=== and we notify about these gaps === +X diff --git a/src/service/rooms/timeline/stitcher/test/testcases/zzz-multiple_events_referring_to_the_same_missing_event_with_more.stitched b/src/service/rooms/timeline/stitcher/test/testcases/zzz-multiple_events_referring_to_the_same_missing_event_with_more.stitched new file mode 100644 index 00000000..120b4c8b --- /dev/null +++ b/src/service/rooms/timeline/stitcher/test/testcases/zzz-multiple_events_referring_to_the_same_missing_event_with_more.stitched @@ -0,0 +1,28 @@ +=== when we receive these events === +# Several events refer to the same missing event, but one refers to others too +A --> X +B --> X +B --> Y +B --> Z +C --> X +=== then we arrange into this order === +# We end up with multiple gaps +-X* +A* +-X,Y,Z* +B* +-X* +C* + +=== when we receive these events === +# When we receive the missing item +X +=== then we arrange into this order === +# It goes into the earliest slot, and the non-empty gap remains +X +A +-Y,Z +B +C +=== and we notify about these gaps === +X diff --git a/src/service/rooms/timeline/stitcher/test/testcases/zzz-multiple_missing_prev_events_turn_into_a_single_gap.stitched b/src/service/rooms/timeline/stitcher/test/testcases/zzz-multiple_missing_prev_events_turn_into_a_single_gap.stitched new file mode 100644 index 00000000..b64b1914 --- /dev/null +++ b/src/service/rooms/timeline/stitcher/test/testcases/zzz-multiple_missing_prev_events_turn_into_a_single_gap.stitched @@ -0,0 +1,9 @@ +=== when we receive these events === +# A refers to multiple missing things +A --> X +A --> Y +A --> Z +=== then we arrange into this order === +# But we only make one gap, with multiple IDs in it +-X,Y,Z* +A* diff --git a/src/service/rooms/timeline/stitcher/test/testcases/zzz-partially_filling_a_gap_leaves_it_before_new_nodes.stitched b/src/service/rooms/timeline/stitcher/test/testcases/zzz-partially_filling_a_gap_leaves_it_before_new_nodes.stitched new file mode 100644 index 00000000..ccb12754 --- /dev/null +++ b/src/service/rooms/timeline/stitcher/test/testcases/zzz-partially_filling_a_gap_leaves_it_before_new_nodes.stitched @@ -0,0 +1,23 @@ +=== when we receive these events === +A +F --> B +F --> C +F --> D +F --> E +=== then we arrange into this order === +# Given a gap that lists several nodes: +A* +-B,C,D,E* +F* +=== when we receive these events === +# When we provide one of the missing events: +C +=== then we arrange into this order === +# Then it is inserted after the gap, and the gap is shrunk: +A +-B,D,E +C +F +=== and we notify about these gaps === +# And we notify about the gap that was updated +C diff --git a/src/service/rooms/timeline/stitcher/test/testcases/zzz-partially_filling_a_gap_with_two_events.stitched b/src/service/rooms/timeline/stitcher/test/testcases/zzz-partially_filling_a_gap_with_two_events.stitched new file mode 100644 index 00000000..81e2fba8 --- /dev/null +++ b/src/service/rooms/timeline/stitcher/test/testcases/zzz-partially_filling_a_gap_with_two_events.stitched @@ -0,0 +1,27 @@ +=== when we receive these events === +# Given an event references multiple missing events +A +F --> B +F --> C +F --> D +F --> E +=== then we arrange into this order === +A* +-B,C,D,E* +F + +=== when we receive these events === +# When we provide some of the missing events +C +E +=== then we arrange into this order === +# Then we insert them after the gap and shrink the list of events in the gap +A +-B,D +C +E +F +=== and we notify about these gaps === +# And we notify about the gap that was updated +C +E diff --git a/src/service/rooms/timeline/stitcher/test/testcases/zzz-received_order_wins_within_a_subgroup_if_no_prev_event_chain.stitched b/src/service/rooms/timeline/stitcher/test/testcases/zzz-received_order_wins_within_a_subgroup_if_no_prev_event_chain.stitched new file mode 100644 index 00000000..26ee8811 --- /dev/null +++ b/src/service/rooms/timeline/stitcher/test/testcases/zzz-received_order_wins_within_a_subgroup_if_no_prev_event_chain.stitched @@ -0,0 +1,16 @@ +=== when we receive these events === +# Everything is after A, but there is no prev_event chain between the others, so +# we use received order. +A +F --> A +C --> A +D --> A +E --> A +B --> A +=== then we arrange into this order === +A* +F* +C* +D* +E* +B* diff --git a/src/service/rooms/timeline/stitcher/test/testcases/zzz-subgroups_are_processed_in_first_received_order.stitched b/src/service/rooms/timeline/stitcher/test/testcases/zzz-subgroups_are_processed_in_first_received_order.stitched new file mode 100644 index 00000000..59b03677 --- /dev/null +++ b/src/service/rooms/timeline/stitcher/test/testcases/zzz-subgroups_are_processed_in_first_received_order.stitched @@ -0,0 +1,16 @@ +=== when we receive these events === +# We preserve the received order where it does not conflict with the prev_events +A +X --> W +Y --> X +Z --> Y +B --> A +C --> B +=== then we arrange into this order === +A* +-W* +X* +Y* +Z* +B* +C*