Import and update the file service application and orizentic #72

Merged
savanni merged 36 commits from file-service into main 2023-10-03 23:50:54 +00:00
6 changed files with 74 additions and 77 deletions
Showing only changes of commit 14f0a74af8 - Show all commits

View File

@ -1,11 +1,3 @@
/*
use iron::headers;
use iron::middleware::Handler;
use iron::modifiers::{Header, Redirect};
use iron::prelude::*;
use iron::response::BodyReader;
use iron::status;
*/
#[macro_use] #[macro_use]
extern crate log; extern crate log;
@ -14,9 +6,8 @@ use http::status::StatusCode;
// use orizentic::{Permissions, ResourceName, Secret}; // use orizentic::{Permissions, ResourceName, Secret};
use build_html::Html; use build_html::Html;
use bytes::Buf; use bytes::Buf;
use futures_util::{Stream, StreamExt}; use futures_util::StreamExt;
use std::{ use std::{
collections::HashMap,
io::Read, io::Read,
net::{IpAddr, Ipv4Addr, SocketAddr}, net::{IpAddr, Ipv4Addr, SocketAddr},
path::PathBuf, path::PathBuf,
@ -24,9 +15,9 @@ use std::{
}; };
use warp::{filters::multipart::Part, Filter}; use warp::{filters::multipart::Part, Filter};
mod cookies; // mod cookies;
mod html; mod html;
mod middleware; // mod middleware;
mod pages; mod pages;
mod store; mod store;
@ -430,7 +421,7 @@ pub async fn main() {
.with(log), .with(log),
); );
*/ */
let server = warp::serve(root.or(upload).with(log)); let server = warp::serve(root.or(upload).or(thumbnail).or(file).or(delete).with(log));
server server
.run(SocketAddr::new(IpAddr::V4(Ipv4Addr::new(0, 0, 0, 0)), 8002)) .run(SocketAddr::new(IpAddr::V4(Ipv4Addr::new(0, 0, 0, 0)), 8002))
.await; .await;

View File

@ -1,6 +1,6 @@
use crate::{ use crate::{
html::*, html::*,
store::{FileHandle, FileId, ReadFileError, Thumbnail}, store::{FileHandle, FileId, ReadFileError},
}; };
use build_html::{self, Container, ContainerType, Html, HtmlContainer}; use build_html::{self, Container, ContainerType, Html, HtmlContainer};

View File

@ -71,7 +71,6 @@ impl TryFrom<String> for PathResolver {
impl TryFrom<&str> for PathResolver { impl TryFrom<&str> for PathResolver {
type Error = PathError; type Error = PathError;
fn try_from(s: &str) -> Result<Self, Self::Error> { fn try_from(s: &str) -> Result<Self, Self::Error> {
let path = Path::new(s);
PathResolver::try_from(Path::new(s)) PathResolver::try_from(Path::new(s))
} }
} }
@ -160,11 +159,12 @@ impl FileHandle {
)?; )?;
let byte_count = content_file.write(&content)?; let byte_count = content_file.write(&content)?;
self.info.size = byte_count; self.info.size = byte_count;
self.info.hash = self.hash_content(&content).as_string();
let mut md_file = std::fs::File::create(self.path.metadata_path())?; let mut md_file = std::fs::File::create(self.path.metadata_path())?;
md_file.write(&serde_json::to_vec(&self.info)?)?; md_file.write(&serde_json::to_vec(&self.info)?)?;
self.write_thumbnail(); self.write_thumbnail()?;
Ok(()) Ok(())
} }
@ -177,12 +177,8 @@ impl FileHandle {
load_content(&self.path.thumbnail_path()?) load_content(&self.path.thumbnail_path()?)
} }
fn hash_content(&self) -> Result<HexString, ReadFileError> { fn hash_content(&self, data: &Vec<u8>) -> HexString {
let mut buf = Vec::new(); HexString::from_bytes(&Sha256::digest(data).to_vec())
let mut file = std::fs::File::open(self.path.file_path()?)?;
file.read_to_end(&mut buf).map_err(ReadFileError::from)?;
Ok(HexString::from_bytes(&Sha256::digest(&buf).to_vec()))
} }
fn write_thumbnail(&self) -> Result<(), WriteFileError> { fn write_thumbnail(&self) -> Result<(), WriteFileError> {
@ -202,20 +198,20 @@ impl FileHandle {
Ok(()) Ok(())
} }
pub fn delete(self) -> Result<(), WriteFileError> { pub fn delete(self) {
std::fs::remove_file( match self.path.thumbnail_path() {
self.path Ok(path) => {
.thumbnail_path() let _ = std::fs::remove_file(path);
.map_err(|_| WriteFileError::NoMetadata)?, }
)?; Err(_) => {}
std::fs::remove_file(self.path.metadata_path())?; };
std::fs::remove_file( match self.path.file_path() {
self.path Ok(path) => {
.file_path() let _ = std::fs::remove_file(path);
.map_err(|_| WriteFileError::NoMetadata)?, }
)?; Err(_) => {}
};
Ok(()) let _ = std::fs::remove_file(self.path.metadata_path());
} }
} }
@ -229,7 +225,7 @@ fn load_content(path: &Path) -> Result<Vec<u8>, ReadFileError> {
#[cfg(test)] #[cfg(test)]
mod test { mod test {
use super::*; use super::*;
use crate::store::utils::FileCleanup; use crate::store::utils::DirCleanup;
use cool_asserts::assert_matches; use cool_asserts::assert_matches;
use std::{convert::TryFrom, path::PathBuf}; use std::{convert::TryFrom, path::PathBuf};
@ -237,6 +233,7 @@ mod test {
fn paths() { fn paths() {
let resolver = PathResolver::try_from("path/82420255-d3c8-4d90-a582-f94be588c70c.png") let resolver = PathResolver::try_from("path/82420255-d3c8-4d90-a582-f94be588c70c.png")
.expect("to have a valid path"); .expect("to have a valid path");
assert_matches!( assert_matches!(
resolver.file_path(), resolver.file_path(),
Ok(path) => assert_eq!(path, PathBuf::from( Ok(path) => assert_eq!(path, PathBuf::from(
@ -257,15 +254,21 @@ mod test {
#[test] #[test]
fn it_opens_a_file() { fn it_opens_a_file() {
let _md = FileCleanup(PathBuf::from("fixtures/.metadata/rawr.png.json")); let _cleanup = DirCleanup(PathBuf::from("var/"));
let _tn = FileCleanup(PathBuf::from("fixtures/.thumbnails/rawr.png"));
FileHandle::new("rawr.png".to_owned(), PathBuf::from("var/")).expect("to succeed"); FileHandle::new("rawr.png".to_owned(), PathBuf::from("var/")).expect("to succeed");
} }
#[test] #[test]
fn it_can_return_a_thumbnail() { fn it_deletes_a_file() {
let _cleanup = DirCleanup(PathBuf::from("var/"));
let f = FileHandle::new("rawr.png".to_owned(), PathBuf::from("var/")).expect("to succeed"); let f = FileHandle::new("rawr.png".to_owned(), PathBuf::from("var/")).expect("to succeed");
f.delete();
}
#[test]
fn it_can_return_a_thumbnail() {
let _cleanup = DirCleanup(PathBuf::from("var/"));
let _ = FileHandle::new("rawr.png".to_owned(), PathBuf::from("var/")).expect("to succeed");
/* /*
assert_eq!( assert_eq!(
f.thumbnail(), f.thumbnail(),
@ -279,13 +282,14 @@ mod test {
#[test] #[test]
fn it_can_return_a_file_stream() { fn it_can_return_a_file_stream() {
let f = FileHandle::new("rawr.png".to_owned(), PathBuf::from("var/")).expect("to succeed"); let _cleanup = DirCleanup(PathBuf::from("var/"));
let _ = FileHandle::new("rawr.png".to_owned(), PathBuf::from("var/")).expect("to succeed");
// f.stream().expect("to succeed"); // f.stream().expect("to succeed");
} }
#[test] #[test]
fn it_raises_an_error_when_file_not_found() { fn it_raises_an_error_when_file_not_found() {
let resolver = PathResolver::try_from("var/rawr.png").expect("a valid path"); let _cleanup = DirCleanup(PathBuf::from("var/"));
match FileHandle::load(&FileId::from("rawr"), &PathBuf::from("var/")) { match FileHandle::load(&FileId::from("rawr"), &PathBuf::from("var/")) {
Err(ReadFileError::FileNotFound(_)) => assert!(true), Err(ReadFileError::FileNotFound(_)) => assert!(true),
_ => assert!(false), _ => assert!(false),

View File

@ -41,15 +41,11 @@ impl FileInfo {
#[cfg(test)] #[cfg(test)]
mod test { mod test {
use super::*; use super::*;
use crate::store::{filehandle::PathResolver, utils::FileCleanup, FileId}; use crate::store::{utils::DirCleanup, FileId};
use std::convert::TryFrom;
#[test] #[test]
fn it_saves_and_loads_metadata() { fn it_saves_and_loads_metadata() {
let resolver = PathResolver::try_from("var/1617654d-a588-4714-b4fa-e00ed0a8a607.png") let _cleanup = DirCleanup(PathBuf::from("var/"));
.expect("a valid path");
let _cleanup = FileCleanup(resolver.metadata_path());
let created = Utc::now(); let created = Utc::now();
let info = FileInfo { let info = FileInfo {
@ -60,9 +56,10 @@ mod test {
hash: "abcdefg".to_owned(), hash: "abcdefg".to_owned(),
extension: "png".to_owned(), extension: "png".to_owned(),
}; };
info.save(resolver.metadata_path()).unwrap(); info.save(PathBuf::from(format!("var/{}", *info.id)))
.unwrap();
let info_ = FileInfo::load(resolver.metadata_path()).unwrap(); let info_ = FileInfo::load(PathBuf::from(format!("var/{}", *info.id))).unwrap();
assert_eq!(info_.size, 23777); assert_eq!(info_.size, 23777);
assert_eq!(info_.created, info.created); assert_eq!(info_.created, info.created);
assert_eq!(info_.file_type, "image/png"); assert_eq!(info_.file_type, "image/png");

View File

@ -1,18 +1,14 @@
use serde::{Deserialize, Serialize}; use serde::{Deserialize, Serialize};
use std::{ use std::collections::HashSet;
ops::Deref, use std::{ops::Deref, path::PathBuf};
path::{Path, PathBuf},
};
use thiserror::Error; use thiserror::Error;
mod filehandle; mod filehandle;
mod fileinfo; mod fileinfo;
mod thumbnail;
pub mod utils; pub mod utils;
pub use filehandle::FileHandle; pub use filehandle::FileHandle;
pub use fileinfo::FileInfo; pub use fileinfo::FileInfo;
pub use thumbnail::Thumbnail;
#[derive(Debug, Error)] #[derive(Debug, Error)]
pub enum WriteFileError { pub enum WriteFileError {
@ -118,8 +114,20 @@ impl Store {
Self { files_root } Self { files_root }
} }
pub fn list_files(&self) -> Result<Vec<FileId>, ReadFileError> { pub fn list_files(&self) -> Result<HashSet<FileId>, ReadFileError> {
unimplemented!() let paths = std::fs::read_dir(&self.files_root)?;
Ok(paths
.into_iter()
.map(|path| {
FileId::from(
path.unwrap()
.path()
.file_stem()
.and_then(|s| s.to_str())
.unwrap(),
)
})
.collect::<HashSet<FileId>>())
} }
pub fn add_file( pub fn add_file(
@ -152,8 +160,7 @@ impl Store {
#[cfg(test)] #[cfg(test)]
mod test { mod test {
use super::*; use super::{utils::DirCleanup, *};
use crate::store::utils::FileCleanup;
use cool_asserts::assert_matches; use cool_asserts::assert_matches;
use std::{collections::HashSet, io::Read}; use std::{collections::HashSet, io::Read};
@ -161,6 +168,8 @@ mod test {
where where
F: FnOnce(Store, FileId), F: FnOnce(Store, FileId),
{ {
let _cleanup = DirCleanup(PathBuf::from("var/"));
let mut buf = Vec::new(); let mut buf = Vec::new();
let mut file = std::fs::File::open("fixtures/rawr.png").unwrap(); let mut file = std::fs::File::open("fixtures/rawr.png").unwrap();
file.read_to_end(&mut buf).unwrap(); file.read_to_end(&mut buf).unwrap();
@ -168,10 +177,6 @@ mod test {
let mut store = Store::new(PathBuf::from("var/")); let mut store = Store::new(PathBuf::from("var/"));
let file_record = store.add_file("rawr.png".to_owned(), buf).unwrap(); let file_record = store.add_file("rawr.png".to_owned(), buf).unwrap();
let _file = FileCleanup(PathBuf::from(format!("var/{}.png", *file_record.id)));
let _md = FileCleanup(PathBuf::from(format!("var/{}.json", *file_record.id)));
let _tn = FileCleanup(PathBuf::from(format!("var/{}.tn.png", *file_record.id)));
test_fn(store, file_record.id); test_fn(store, file_record.id);
} }
@ -191,6 +196,7 @@ mod test {
#[test] #[test]
fn sets_up_metadata_for_file() { fn sets_up_metadata_for_file() {
with_file(|store, id| { with_file(|store, id| {
assert!(PathBuf::from(format!("var/{}.png", *id)).exists());
let info = store.get_metadata(&id).expect("to retrieve the metadata"); let info = store.get_metadata(&id).expect("to retrieve the metadata");
assert_matches!(info, FileInfo { size, file_type, hash, extension, .. } => { assert_matches!(info, FileInfo { size, file_type, hash, extension, .. } => {
@ -229,6 +235,7 @@ mod test {
let resolvers = store.list_files().expect("file listing to succeed"); let resolvers = store.list_files().expect("file listing to succeed");
let ids = resolvers.into_iter().collect::<HashSet<FileId>>(); let ids = resolvers.into_iter().collect::<HashSet<FileId>>();
println!("ids: {:?}", ids);
assert_eq!(ids.len(), 1); assert_eq!(ids.len(), 1);
}); });
} }

View File

@ -1,17 +1,15 @@
use std::path::{Path, PathBuf}; use std::{ffi::OsStr, path::PathBuf};
pub struct FileCleanup(pub PathBuf); pub struct DirCleanup(pub PathBuf);
impl Drop for FileCleanup { impl Drop for DirCleanup {
fn drop(&mut self) { fn drop(&mut self) {
let _ = std::fs::remove_file(&self.0); let files = std::fs::read_dir(&self.0).unwrap();
for file in files {
let filename = file.unwrap().path();
if filename.file_name() != Some(&OsStr::new(".placeholder")) {
let _ = std::fs::remove_file(filename);
}
}
} }
} }
pub fn append_extension(path: &Path, extra_ext: &str) -> PathBuf {
let ext_ = match path.extension() {
None => String::from(extra_ext),
Some(ext) => [ext.to_string_lossy(), std::borrow::Cow::from(extra_ext)].join("."),
};
path.with_extension(ext_)
}