Improve API

backend/reconcile/src/tokenizer/token.rs

@@ -2,8 +2,12 @@
 use serde::{Deserialize, Serialize};
 
 /// A token is a string that has been normalised in some way.
-/// The normalised form is used for comparison, while the original form is used
-/// for applying `Operation`-s.
+///
+/// It's UTF-8 compatible.
+///
+/// A token consists of the normalised form is used for comparison, and the
+/// original form used for subsequently applying `Operation`-s to a text
+/// document.
 #[cfg_attr(feature = "serde", derive(Serialize, Deserialize))]
 #[derive(Debug, Clone)]
 pub struct Token<T>
@@ -11,18 +15,20 @@ where
     T: PartialEq + Clone + std::fmt::Debug,
 {
     /// The normalised form of the token used deriving the diff.
-    pub normalised: T,
+    normalised: T,
 
     /// The original string, that should be inserted or deleted in the document.
     original: String,
 
-    /// Whether the token is joinable with the previous token.
-    is_left_joinable: bool,
+    /// Whether the token is semantically joinable with the previous token.
+    pub is_left_joinable: bool,
 
-    /// Whether the token is joinable with the next token.
-    is_right_joinable: bool,
+    /// Whether the token is semantically joinable with the next token.
+    pub is_right_joinable: bool,
 }
 
+/// Trivial implementation of Token when the normalised form is the same as the
+/// original string.
 impl From<&str> for Token<String> {
     fn from(text: &str) -> Self { Token::new(text.to_owned(), text.to_owned(), true, true) }
 }
@@ -47,13 +53,11 @@ where
 
     pub fn original(&self) -> &str { &self.original }
 
+    pub fn set_normalised(&mut self, normalised: T) { self.normalised = normalised; }
+
     pub fn normalised(&self) -> &T { &self.normalised }
 
     pub fn get_original_length(&self) -> usize { self.original.chars().count() }
-
-    pub fn get_is_left_joinable(&self) -> bool { self.is_left_joinable }
-
-    pub fn get_is_right_joinable(&self) -> bool { self.is_right_joinable }
 }
 
 impl<T> PartialEq for Token<T>