savanni
/
monorepo
1
0
Fork 0
Code Issues 68 Pull Requests Projects 6 Releases Wiki Activity

Bind the ID to the record instead of keeping them separate

This commit is contained in:
Savanni D'Gerinel 2023-12-27 16:13:47 -05:00
parent d2f4ec97c0
commit 149587f0bd
4 changed files with 131 additions and 83 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
emseries/src/lib.rs
View File

@ -76,4 +76,4 @@ mod types;
pub use criteria::*; pub use criteria::*;
pub use series::Series; pub use series::Series;
pub use types::{EmseriesReadError, EmseriesWriteError, RecordId, Recordable, Timestamp}; pub use types::{EmseriesReadError, EmseriesWriteError, Record, RecordId, Recordable, Timestamp};

93
emseries/src/series.rs
View File

@ -18,6 +18,7 @@ use serde::de::DeserializeOwned;
use serde::ser::Serialize; use serde::ser::Serialize;
use std::cmp::Ordering; use std::cmp::Ordering;
use std::collections::HashMap; use std::collections::HashMap;
use std::convert::TryFrom;
use std::fs::File; use std::fs::File;
use std::fs::OpenOptions; use std::fs::OpenOptions;
use std::io::{BufRead, BufReader, LineWriter, Write}; use std::io::{BufRead, BufReader, LineWriter, Write};
@ -26,6 +27,43 @@ use std::iter::Iterator;
use criteria::Criteria; use criteria::Criteria;
use types::{EmseriesReadError, EmseriesWriteError, Record, RecordId, Recordable}; use types::{EmseriesReadError, EmseriesWriteError, Record, RecordId, Recordable};
// A RecordOnDisk, a private data structure, is useful for handling all of the on-disk
// representations of a record. Unlike [Record], this one can accept an empty data value to
// represent that the data may have been deleted. This is not made public because, so far as the
// user is concerned, any record in the system must have data associated with it.
#[derive(Clone, Deserialize, Serialize)]
struct RecordOnDisk<T: Clone + Recordable> {
id: RecordId,
data: Option<T>,
}
/*
impl<T> FromStr for RecordOnDisk<T>
where
T: Clone + Recordable + DeserializeOwned + Serialize,
{
type Err = EmseriesReadError;
fn from_str(line: &str) -> Result<Self, Self::Err> {
serde_json::from_str(line).map_err(EmseriesReadError::JSONParseError)
}
}
*/
impl<T: Clone + Recordable> TryFrom<RecordOnDisk<T>> for Record<T> {
type Error = EmseriesReadError;
fn try_from(disk_record: RecordOnDisk<T>) -> Result<Self, Self::Error> {
match disk_record.data {
Some(data) => Ok(Record {
id: disk_record.id,
data,
}),
None => Err(Self::Error::RecordDeleted(disk_record.id)),
}
}
}
/// An open time series database. /// An open time series database.
/// ///
/// Any given database can store only one data type, T. The data type must be determined when the /// Any given database can store only one data type, T. The data type must be determined when the
@ -33,7 +71,7 @@ use types::{EmseriesReadError, EmseriesWriteError, Record, RecordId, Recordable}
pub struct Series<T: Clone + Recordable + DeserializeOwned + Serialize> { pub struct Series<T: Clone + Recordable + DeserializeOwned + Serialize> {
//path: String, //path: String,
writer: LineWriter<File>, writer: LineWriter<File>,
records: HashMap<RecordId, T>, records: HashMap<RecordId, Record<T>>,
} }
impl<T> Series<T> impl<T> Series<T>
@ -62,20 +100,18 @@ where
} }
/// Load a file and return all of the records in it. /// Load a file and return all of the records in it.
fn load_file(f: &File) -> Result<HashMap<RecordId, T>, EmseriesReadError> { fn load_file(f: &File) -> Result<HashMap<RecordId, Record<T>>, EmseriesReadError> {
let mut records: HashMap<RecordId, T> = HashMap::new(); let mut records: HashMap<RecordId, Record<T>> = HashMap::new();
let reader = BufReader::new(f); let reader = BufReader::new(f);
for line in reader.lines() { for line in reader.lines() {
match line { match line {
Ok(line_) => { Ok(line_) => {
/* Can't create a JSONParseError because I can't actually create the underlying error. match serde_json::from_str::<RecordOnDisk<T>>(line_.as_ref())
fail_point!("parse-line", Err(Error::JSONParseError())) .map_err(EmseriesReadError::JSONParseError)
*/ .and_then(|record| Record::try_from(record))
match line_.parse::<Record<T>>() { {
Ok(record) => match record.data { Ok(record) => records.insert(record.id.clone(), record.clone()),
Some(val) => records.insert(record.id.clone(), val), Err(EmseriesReadError::RecordDeleted(id)) => records.remove(&id),
None => records.remove(&record.id.clone()),
},
Err(err) => return Err(err), Err(err) => return Err(err),
}; };
} }
@ -89,16 +125,21 @@ where
/// returned. /// returned.
pub fn put(&mut self, entry: T) -> Result<RecordId, EmseriesWriteError> { pub fn put(&mut self, entry: T) -> Result<RecordId, EmseriesWriteError> {
let uuid = RecordId::default(); let uuid = RecordId::default();
self.update(uuid.clone(), entry).map(|_| uuid) let record = Record {
id: uuid.clone(),
data: entry,
};
self.update(record)?;
Ok(uuid)
} }
/// Update an existing record. The [RecordId] of the record passed into this function must match /// Update an existing record. The [RecordId] of the record passed into this function must match
/// the [RecordId] of a record already in the database. /// the [RecordId] of a record already in the database.
pub fn update(&mut self, uuid: RecordId, entry: T) -> Result<(), EmseriesWriteError> { pub fn update(&mut self, record: Record<T>) -> Result<(), EmseriesWriteError> {
self.records.insert(uuid.clone(), entry.clone()); self.records.insert(record.id.clone(), record.clone());
let write_res = match serde_json::to_string(&Record { let write_res = match serde_json::to_string(&RecordOnDisk {
id: uuid, id: record.id,
data: Some(entry), data: Some(record.data),
}) { }) {
Ok(rec_str) => self Ok(rec_str) => self
.writer .writer
@ -124,7 +165,7 @@ where
}; };
self.records.remove(uuid); self.records.remove(uuid);
let rec: Record<T> = Record { let rec: RecordOnDisk<T> = RecordOnDisk {
id: uuid.clone(), id: uuid.clone(),
data: None, data: None,
}; };
@ -138,8 +179,8 @@ where
} }
/// Get all of the records in the database. /// Get all of the records in the database.
pub fn records(&self) -> impl Iterator<Item = (&RecordId, &T)> { pub fn records(&self) -> impl Iterator<Item = &Record<T>> {
self.records.iter() self.records.values()
} }
/* The point of having Search is so that a lot of internal optimizations can happen once the /* The point of having Search is so that a lot of internal optimizations can happen once the
@ -148,24 +189,24 @@ where
pub fn search<'s>( pub fn search<'s>(
&'s self, &'s self,
criteria: impl Criteria + 's, criteria: impl Criteria + 's,
) -> impl Iterator<Item = (&'s RecordId, &'s T)> + 's { ) -> impl Iterator<Item = &'s Record<T>> + 's {
self.records().filter(move |&tr| criteria.apply(tr.1)) self.records().filter(move |&tr| criteria.apply(&tr.data))
} }
/// Perform a search and sort the resulting records based on the comparison. /// Perform a search and sort the resulting records based on the comparison.
pub fn search_sorted<'s, C, CMP>(&'s self, criteria: C, compare: CMP) -> Vec<(&RecordId, &T)> pub fn search_sorted<'s, C, CMP>(&'s self, criteria: C, compare: CMP) -> Vec<&'s Record<T>>
where where
C: Criteria + 's, C: Criteria + 's,
CMP: FnMut(&(&RecordId, &T), &(&RecordId, &T)) -> Ordering, CMP: FnMut(&&Record<T>, &&Record<T>) -> Ordering,
{ {
let search_iter = self.search(criteria); let search_iter = self.search(criteria);
let mut records: Vec<(&RecordId, &T)> = search_iter.collect(); let mut records: Vec<&Record<T>> = search_iter.collect();
records.sort_by(compare); records.sort_by(compare);
records records
} }
/// Get an exact record from the database based on unique id. /// Get an exact record from the database based on unique id.
pub fn get(&self, uuid: &RecordId) -> Option<T> { pub fn get(&self, uuid: &RecordId) -> Option<Record<T>> {
self.records.get(uuid).cloned() self.records.get(uuid).cloned()
} }

36
emseries/src/types.rs
View File

@ -28,6 +28,9 @@ pub enum EmseriesReadError {
#[error("Error parsing JSON: {0}")] #[error("Error parsing JSON: {0}")]
JSONParseError(serde_json::error::Error), JSONParseError(serde_json::error::Error),
#[error("Record was deleted")]
RecordDeleted(RecordId),
/// Indicates a general IO error /// Indicates a general IO error
#[error("IO Error: {0}")] #[error("IO Error: {0}")]
IOError(io::Error), IOError(io::Error),
@ -171,28 +174,31 @@ impl fmt::Display for RecordId {
} }
} }
/// Every record contains a unique ID and then the primary data, which itself must implementd the /// A record represents data that actually exists in the database. Users cannot make the record
/// Recordable trait. /// directly, as the database will create them.
#[derive(Clone, Deserialize, Serialize)] #[derive(Clone, Debug, PartialEq, Eq, Deserialize, Serialize)]
pub struct Record<T: Clone + Recordable> { pub struct Record<T: Clone + Recordable> {
pub id: RecordId, pub(crate) id: RecordId,
pub data: Option<T>, pub data: T,
} }
impl<T> str::FromStr for Record<T> impl<T: Clone + Recordable> Record<T> {
where pub fn date(&self) -> NaiveDate {
T: Clone + Recordable + DeserializeOwned + Serialize, match self.data.timestamp() {
{ Timestamp::DateTime(dt) => dt.date_naive(),
type Err = EmseriesReadError; Timestamp::Date(dt) => dt,
}
}
fn from_str(line: &str) -> Result<Self, Self::Err> { pub fn timestamp(&self) -> Timestamp {
serde_json::from_str(line).map_err(EmseriesReadError::JSONParseError) self.data.timestamp()
} }
} }
#[cfg(test)] #[cfg(test)]
mod test { mod test {
extern crate dimensioned; extern crate dimensioned;
extern crate serde_json; extern crate serde_json;
use self::dimensioned::si::{Kilogram, KG}; use self::dimensioned::si::{Kilogram, KG};
@ -239,6 +245,7 @@ mod test {
); );
} }
/*
#[ignore] #[ignore]
fn v_alpha_serialization() { fn v_alpha_serialization() {
const WEIGHT_ENTRY: &str = "{\"data\":{\"weight\":77.79109},\"date\":\"2003-11-10\",\"id\":\"3330c5b0-783f-4919-b2c4-8169c38f65ff\"}"; const WEIGHT_ENTRY: &str = "{\"data\":{\"weight\":77.79109},\"date\":\"2003-11-10\",\"id\":\"3330c5b0-783f-4919-b2c4-8169c38f65ff\"}";
@ -252,12 +259,13 @@ mod test {
); );
assert_eq!( assert_eq!(
rec.data, rec.data,
Some(WeightRecord { WeightRecord {
date: NaiveDate::from_ymd_opt(2003, 11, 10).unwrap(), date: NaiveDate::from_ymd_opt(2003, 11, 10).unwrap(),
weight: Weight(77.79109 * KG), weight: Weight(77.79109 * KG),
}) }
); );
} }
*/
#[test] #[test]
fn serialization_output() { fn serialization_output() {

83
emseries/tests/test_io.rs
View File

@ -143,9 +143,9 @@ mod test {
.with_timezone(&FixedOffset::east_opt(0).unwrap()) .with_timezone(&FixedOffset::east_opt(0).unwrap())
) )
); );
assert_eq!(tr.duration, Duration(11040.0 * S)); assert_eq!(tr.data.duration, Duration(11040.0 * S));
assert_eq!(tr.comments, String::from("long time ago")); assert_eq!(tr.data.comments, String::from("long time ago"));
assert_eq!(tr, trips[0]); assert_eq!(tr.data, trips[0]);
} }
} }
}) })
@ -162,7 +162,7 @@ mod test {
ts.put(trip.clone()).expect("expect a successful put"); ts.put(trip.clone()).expect("expect a successful put");
} }
let v: Vec<(&RecordId, &BikeTrip)> = ts let v: Vec<&Record<BikeTrip>> = ts
.search(exact_time(Timestamp::DateTime( .search(exact_time(Timestamp::DateTime(
UTC.with_ymd_and_hms(2011, 10, 31, 0, 0, 0) UTC.with_ymd_and_hms(2011, 10, 31, 0, 0, 0)
.unwrap() .unwrap()
@ -170,7 +170,7 @@ mod test {
))) )))
.collect(); .collect();
assert_eq!(v.len(), 1); assert_eq!(v.len(), 1);
assert_eq!(*v[0].1, trips[1]); assert_eq!(v[0].data, trips[1]);
}) })
} }
@ -185,7 +185,7 @@ mod test {
ts.put(trip.clone()).expect("expect a successful put"); ts.put(trip.clone()).expect("expect a successful put");
} }
let v: Vec<(&RecordId, &BikeTrip)> = ts.search_sorted( let v: Vec<&Record<BikeTrip>> = ts.search_sorted(
time_range( time_range(
Timestamp::DateTime( Timestamp::DateTime(
UTC.with_ymd_and_hms(2011, 10, 31, 0, 0, 0) UTC.with_ymd_and_hms(2011, 10, 31, 0, 0, 0)
@ -200,12 +200,12 @@ mod test {
), ),
true, true,
), ),
|l, r| l.1.timestamp().cmp(&r.1.timestamp()), |l, r| l.timestamp().cmp(&r.timestamp()),
); );
assert_eq!(v.len(), 3); assert_eq!(v.len(), 3);
assert_eq!(*v[0].1, trips[1]); assert_eq!(v[0].data, trips[1]);
assert_eq!(*v[1].1, trips[2]); assert_eq!(v[1].data, trips[2]);
assert_eq!(*v[2].1, trips[3]); assert_eq!(v[2].data, trips[3]);
}) })
} }
@ -226,7 +226,7 @@ mod test {
{ {
let ts: Series<BikeTrip> = let ts: Series<BikeTrip> =
Series::open(&path).expect("expect the time series to open correctly"); Series::open(&path).expect("expect the time series to open correctly");
let v: Vec<(&RecordId, &BikeTrip)> = ts.search_sorted( let v: Vec<&Record<BikeTrip>> = ts.search_sorted(
time_range( time_range(
Timestamp::DateTime( Timestamp::DateTime(
UTC.with_ymd_and_hms(2011, 10, 31, 0, 0, 0) UTC.with_ymd_and_hms(2011, 10, 31, 0, 0, 0)
@ -241,12 +241,12 @@ mod test {
), ),
true, true,
), ),
|l, r| l.1.timestamp().cmp(&r.1.timestamp()), |l, r| l.timestamp().cmp(&r.timestamp()),
); );
assert_eq!(v.len(), 3); assert_eq!(v.len(), 3);
assert_eq!(*v[0].1, trips[1]); assert_eq!(v[0].data, trips[1]);
assert_eq!(*v[1].1, trips[2]); assert_eq!(v[1].data, trips[2]);
assert_eq!(*v[2].1, trips[3]); assert_eq!(v[2].data, trips[3]);
} }
}) })
} }
@ -268,7 +268,7 @@ mod test {
{ {
let mut ts: Series<BikeTrip> = let mut ts: Series<BikeTrip> =
Series::open(&path).expect("expect the time series to open correctly"); Series::open(&path).expect("expect the time series to open correctly");
let v: Vec<(&RecordId, &BikeTrip)> = ts.search_sorted( let v: Vec<&Record<BikeTrip>> = ts.search_sorted(
time_range( time_range(
Timestamp::DateTime( Timestamp::DateTime(
UTC.with_ymd_and_hms(2011, 10, 31, 0, 0, 0) UTC.with_ymd_and_hms(2011, 10, 31, 0, 0, 0)
@ -284,11 +284,11 @@ mod test {
.into(), .into(),
true, true,
), ),
|l, r| l.1.timestamp().cmp(&r.1.timestamp()), |l, r| l.timestamp().cmp(&r.timestamp()),
); );
assert_eq!(v.len(), 2); assert_eq!(v.len(), 2);
assert_eq!(*v[0].1, trips[1]); assert_eq!(v[0].data, trips[1]);
assert_eq!(*v[1].1, trips[2]); assert_eq!(v[1].data, trips[2]);
ts.put(trips[3].clone()).expect("expect a successful put"); ts.put(trips[3].clone()).expect("expect a successful put");
ts.put(trips[4].clone()).expect("expect a successful put"); ts.put(trips[4].clone()).expect("expect a successful put");
} }
@ -296,7 +296,7 @@ mod test {
{ {
let ts: Series<BikeTrip> = let ts: Series<BikeTrip> =
Series::open(&path).expect("expect the time series to open correctly"); Series::open(&path).expect("expect the time series to open correctly");
let v: Vec<(&RecordId, &BikeTrip)> = ts.search_sorted( let v: Vec<&Record<BikeTrip>> = ts.search_sorted(
time_range( time_range(
Timestamp::DateTime( Timestamp::DateTime(
UTC.with_ymd_and_hms(2011, 10, 31, 0, 0, 0) UTC.with_ymd_and_hms(2011, 10, 31, 0, 0, 0)
@ -311,13 +311,13 @@ mod test {
), ),
true, true,
), ),
|l, r| l.1.timestamp().cmp(&r.1.timestamp()), |l, r| l.timestamp().cmp(&r.timestamp()),
); );
assert_eq!(v.len(), 4); assert_eq!(v.len(), 4);
assert_eq!(*v[0].1, trips[1]); assert_eq!(v[0].data, trips[1]);
assert_eq!(*v[1].1, trips[2]); assert_eq!(v[1].data, trips[2]);
assert_eq!(*v[2].1, trips[3]); assert_eq!(v[2].data, trips[3]);
assert_eq!(*v[3].1, trips[4]); assert_eq!(v[3].data, trips[4]);
} }
}) })
} }
@ -337,9 +337,8 @@ mod test {
match ts.get(&trip_id) { match ts.get(&trip_id) {
None => assert!(false, "record not found"), None => assert!(false, "record not found"),
Some(mut trip) => { Some(mut trip) => {
trip.distance = Distance(50000.0 * M); trip.data.distance = Distance(50000.0 * M);
ts.update(trip_id.clone(), trip) ts.update(trip).expect("expect record to update");
.expect("expect record to update");
} }
}; };
@ -347,12 +346,12 @@ mod test {
None => assert!(false, "record not found"), None => assert!(false, "record not found"),
Some(trip) => { Some(trip) => {
assert_eq!( assert_eq!(
trip.datetime, trip.data.datetime,
UTC.with_ymd_and_hms(2011, 11, 02, 0, 0, 0).unwrap() UTC.with_ymd_and_hms(2011, 11, 02, 0, 0, 0).unwrap()
); );
assert_eq!(trip.distance, Distance(50000.0 * M)); assert_eq!(trip.data.distance, Distance(50000.0 * M));
assert_eq!(trip.duration, Duration(7020.0 * S)); assert_eq!(trip.data.duration, Duration(7020.0 * S));
assert_eq!(trip.comments, String::from("Do Some Distance!")); assert_eq!(trip.data.comments, String::from("Do Some Distance!"));
} }
} }
}) })
@ -374,8 +373,8 @@ mod test {
match ts.get(&trip_id) { match ts.get(&trip_id) {
None => assert!(false, "record not found"), None => assert!(false, "record not found"),
Some(mut trip) => { Some(mut trip) => {
trip.distance = Distance(50000.0 * M); trip.data.distance = Distance(50000.0 * M);
ts.update(trip_id, trip).expect("expect record to update"); ts.update(trip).expect("expect record to update");
} }
}; };
} }
@ -384,10 +383,10 @@ mod test {
let ts: Series<BikeTrip> = let ts: Series<BikeTrip> =
Series::open(&path).expect("expect the time series to open correctly"); Series::open(&path).expect("expect the time series to open correctly");
let trips: Vec<(&RecordId, &BikeTrip)> = ts.records().collect(); let trips: Vec<&Record<BikeTrip>> = ts.records().collect();
assert_eq!(trips.len(), 3); assert_eq!(trips.len(), 3);
let trips: Vec<(&RecordId, &BikeTrip)> = ts let trips: Vec<&Record<BikeTrip>> = ts
.search(exact_time(Timestamp::DateTime( .search(exact_time(Timestamp::DateTime(
UTC.with_ymd_and_hms(2011, 11, 02, 0, 0, 0) UTC.with_ymd_and_hms(2011, 11, 02, 0, 0, 0)
.unwrap() .unwrap()
@ -396,14 +395,14 @@ mod test {
.collect(); .collect();
assert_eq!(trips.len(), 1); assert_eq!(trips.len(), 1);
assert_eq!( assert_eq!(
trips[0].1.datetime, trips[0].data.datetime,
UTC.with_ymd_and_hms(2011, 11, 02, 0, 0, 0) UTC.with_ymd_and_hms(2011, 11, 02, 0, 0, 0)
.unwrap() .unwrap()
.with_timezone(&FixedOffset::east_opt(0).unwrap()) .with_timezone(&FixedOffset::east_opt(0).unwrap())
); );
assert_eq!(trips[0].1.distance, Distance(50000.0 * M)); assert_eq!(trips[0].data.distance, Distance(50000.0 * M));
assert_eq!(trips[0].1.duration, Duration(7020.0 * S)); assert_eq!(trips[0].data.duration, Duration(7020.0 * S));
assert_eq!(trips[0].1.comments, String::from("Do Some Distance!")); assert_eq!(trips[0].data.comments, String::from("Do Some Distance!"));
} }
}) })
} }
@ -421,14 +420,14 @@ mod test {
ts.put(trips[2].clone()).expect("expect a successful put"); ts.put(trips[2].clone()).expect("expect a successful put");
ts.delete(&trip_id).expect("successful delete"); ts.delete(&trip_id).expect("successful delete");
let recs: Vec<(&RecordId, &BikeTrip)> = ts.records().collect(); let recs: Vec<&Record<BikeTrip>> = ts.records().collect();
assert_eq!(recs.len(), 2); assert_eq!(recs.len(), 2);
} }
{ {
let ts: Series<BikeTrip> = let ts: Series<BikeTrip> =
Series::open(&path).expect("expect the time series to open correctly"); Series::open(&path).expect("expect the time series to open correctly");
let recs: Vec<(&RecordId, &BikeTrip)> = ts.records().collect(); let recs: Vec<&Record<BikeTrip>> = ts.records().collect();
assert_eq!(recs.len(), 2); assert_eq!(recs.len(), 2);
} }
}) })