From d658747202fe823e6cf329ecd7195220208fb7d6 Mon Sep 17 00:00:00 2001 From: Savanni D'Gerinel Date: Fri, 22 Sep 2023 20:03:58 -0400 Subject: [PATCH] Start ripping out lots of infrastructure Much of the infrastructure is old and seems to be based on some assumptions about how Iron handled multipart posts. I don't understand how much of this works, so I'm slowly ripping parts out and rebuilding how the separation of concerns works. --- Cargo.lock | 2 + file-service/.gitignore | 1 + file-service/.ignore | 1 + file-service/Cargo.toml | 3 + file-service/src/lib/file.rs | 120 +++++++++++++++++++++++++---- file-service/src/lib/fileinfo.rs | 73 +++++++++++++----- file-service/src/lib/mod.rs | 124 ++++++++++++++++++++++++++---- file-service/src/lib/thumbnail.rs | 25 +++--- file-service/src/main.rs | 57 ++++++++++++-- file-service/src/pages.rs | 10 +-- 10 files changed, 349 insertions(+), 67 deletions(-) create mode 100644 file-service/.gitignore create mode 100644 file-service/.ignore diff --git a/Cargo.lock b/Cargo.lock index 90aa00d..3955b06 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -714,7 +714,9 @@ name = "file-service" version = "0.1.0" dependencies = [ "build_html", + "bytes", "chrono", + "futures-util", "hex-string", "http", "image 0.23.14", diff --git a/file-service/.gitignore b/file-service/.gitignore new file mode 100644 index 0000000..116caa1 --- /dev/null +++ b/file-service/.gitignore @@ -0,0 +1 @@ +fixtures diff --git a/file-service/.ignore b/file-service/.ignore new file mode 100644 index 0000000..116caa1 --- /dev/null +++ b/file-service/.ignore @@ -0,0 +1 @@ +fixtures diff --git a/file-service/Cargo.toml b/file-service/Cargo.toml index 6baf42a..103fdb6 100644 --- a/file-service/Cargo.toml +++ b/file-service/Cargo.toml @@ -29,3 +29,6 @@ uuid = { version = "0.4", features = [ "serde", "v4" ] } warp = { version = "0.3" } pretty_env_logger = { version = "0.5" } log = { version = "0.4" } +bytes = { version = "1" } +futures-util = { version = "0.3" } + diff --git a/file-service/src/lib/file.rs b/file-service/src/lib/file.rs index fad9e08..9a78543 100644 --- a/file-service/src/lib/file.rs +++ b/file-service/src/lib/file.rs @@ -1,9 +1,17 @@ -use super::fileinfo::FileInfo; -use super::thumbnail::Thumbnail; -use std::fs::{copy, read_dir, remove_file}; -use std::path::{Path, PathBuf}; +use super::{ + fileinfo::FileInfo, thumbnail::Thumbnail, FileId, FileRoot, PathResolver, ReadFileError, + WriteFileError, +}; +use chrono::prelude::*; +use std::{ + fs::{copy, read_dir, remove_file}, + io::{Read, Write}, + path::{Path, PathBuf}, +}; use thiserror::Error; +use uuid::Uuid; +/* #[derive(Error, Debug)] pub enum FileError { #[error("not implemented")] @@ -30,21 +38,79 @@ pub enum FileError { #[error("UTF8 Error")] UTF8Error(#[from] std::str::Utf8Error), } +*/ /// One file in the database, complete with the path of the file and information about the /// thumbnail of the file. #[derive(Debug)] pub struct File { - info: FileInfo, - tn: Thumbnail, - root: PathBuf, + pub id: FileId, + path: PathResolver, + pub info: FileInfo, + tn: Option, } impl File { + /// Create a new entry in the database + pub fn new(filename: String, context: CTX) -> Result { + let id = FileId::from(Uuid::new_v4().hyphenated().to_string()); + + let mut path = context.root(); + path.push((*id).to_owned()); + let path = PathResolver(path); + + let file_type = mime_guess::from_ext(&filename) + .first_or_text_plain() + .essence_str() + .to_owned(); + + let info = FileInfo { + id: (*id).to_owned(), + size: 0, + created: Utc::now(), + file_type, + hash: "".to_owned(), + root: context.root().to_owned(), + }; + + let mut md_file = std::fs::File::create(path.metadata_path())?; + md_file.write(&serde_json::to_vec(&info)?)?; + + Ok(Self { + id, + path, + info, + tn: None, + }) + } + + pub fn load(id: FileId, context: CTX) -> Result { + let mut path = context.root(); + path.push((*id).to_owned()); + let path = PathResolver(path); + + Ok(Self { + id: id.clone(), + path, + info: FileInfo::load(id, context)?, + tn: None, + }) + } + + pub fn set_content(&self, content: Vec) -> Result<(), WriteFileError> { + let mut content_file = std::fs::File::create(self.path.file_path())?; + content_file.write(&content)?; + Ok(()) + } + + pub fn content(&self) -> Result, ReadFileError> { + unimplemented!() + } + + /* pub fn new( id: &str, root: &Path, - temp_path: &PathBuf, filename: &Option, ) -> Result { let mut dest_path = PathBuf::from(root); @@ -136,6 +202,7 @@ impl File { self.tn.delete()?; self.info.delete() } + */ } #[cfg(test)] @@ -144,17 +211,34 @@ mod test { use crate::lib::utils::FileCleanup; use std::path::{Path, PathBuf}; + struct FileContext(PathBuf); + + impl FileRoot for FileContext { + fn root(&self) -> PathBuf { + self.0.clone() + } + } + #[test] fn it_opens_a_file() { let _md = FileCleanup(PathBuf::from("fixtures/.metadata/rawr.png.json")); let _tn = FileCleanup(PathBuf::from("fixtures/.thumbnails/rawr.png")); - File::open("rawr.png", Path::new("fixtures/")).expect("to succeed"); + File::new( + "rawr.png".to_owned(), + FileContext(PathBuf::from("fixtures/")), + ) + .expect("to succeed"); } #[test] fn it_can_return_a_thumbnail() { - let f = File::open("rawr.png", Path::new("fixtures/")).expect("to succeed"); + let f = File::new( + "rawr.png".to_owned(), + FileContext(PathBuf::from("fixtures/")), + ) + .expect("to succeed"); + /* assert_eq!( f.thumbnail(), Thumbnail { @@ -162,18 +246,26 @@ mod test { root: PathBuf::from("fixtures/"), }, ); + */ } #[test] fn it_can_return_a_file_stream() { - let f = File::open("rawr.png", Path::new("fixtures/")).expect("to succeed"); - f.stream().expect("to succeed"); + let f = File::new( + "rawr.png".to_owned(), + FileContext(PathBuf::from("fixtures/")), + ) + .expect("to succeed"); + // f.stream().expect("to succeed"); } #[test] fn it_raises_an_error_when_file_not_found() { - match File::open("garbage", Path::new("fixtures/")) { - Err(Error::FileNotFound(_)) => assert!(true), + match File::load( + FileId::from("rawr.png"), + FileContext(PathBuf::from("fixtures/")), + ) { + Err(ReadFileError::FileNotFound) => assert!(true), _ => assert!(false), } } diff --git a/file-service/src/lib/fileinfo.rs b/file-service/src/lib/fileinfo.rs index b3e6d49..5ddf3dc 100644 --- a/file-service/src/lib/fileinfo.rs +++ b/file-service/src/lib/fileinfo.rs @@ -7,7 +7,8 @@ use std::fs::remove_file; use std::io::{Read, Write}; use std::path::{Path, PathBuf}; -use crate::{append_extension, FileError}; +use super::{FileId, FileRoot, PathResolver, ReadFileError}; +use crate::append_extension; #[derive(Clone, Debug, Serialize, Deserialize)] pub struct FileInfo { @@ -18,17 +19,34 @@ pub struct FileInfo { pub hash: String, #[serde(skip)] - root: PathBuf, + pub root: PathBuf, } impl FileInfo { - pub fn save(&self, root: &Path) -> Result<(), FileError> { - let ser = serde_json::to_string(self).unwrap(); - std::fs::File::create(FileInfo::metadata_path(&self.id, root)) - .and_then(|mut stream| stream.write(ser.as_bytes()).map(|_| (()))) - .map_err(FileError::from) + pub fn load(id: FileId, context: CTX) -> Result { + let mut content: Vec = Vec::new(); + let mut file = std::fs::File::open(Self::path(id, context))?; + file.read_to_end(&mut content)?; + let js = serde_json::from_slice(&content)?; + + Ok(js) } + pub fn path(id: FileId, context: CTX) -> PathBuf { + let mut path = context.root(); + path.push((*id).to_owned()); + path.set_extension("json"); + path + } + + pub fn save(&self, root: &PathResolver) -> Result<(), ReadFileError> { + let ser = serde_json::to_string(self).unwrap(); + std::fs::File::create(root.metadata_path()) + .and_then(|mut stream| stream.write(ser.as_bytes()).map(|_| (()))) + .map_err(ReadFileError::from) + } + + /* pub fn open(id: &str, root: &Path) -> Result { let mut buf = Vec::new(); @@ -44,23 +62,24 @@ impl FileInfo { serde_json::from_str(&str_repr).map_err(FileError::from) } + */ - pub fn from_path(path: &Path) -> Result { + pub fn from_path(path: &Path) -> Result { match (path.is_file(), path.is_dir()) { - (false, false) => Err(FileError::FileNotFound(PathBuf::from(path))), - (false, true) => Err(FileError::NotAFile(PathBuf::from(path))), + (false, false) => Err(ReadFileError::FileNotFound), + (false, true) => Err(ReadFileError::NotAFile), (true, _) => Ok(()), }?; - let metadata = path.metadata().map_err(FileError::IOError)?; + let metadata = path.metadata().map_err(ReadFileError::IOError)?; let id = path .file_name() .map(|s| String::from(s.to_string_lossy())) - .ok_or(FileError::NotAFile(PathBuf::from(path)))?; + .ok_or(ReadFileError::NotAFile)?; let created = metadata .created() .map(|m| DateTime::from(m)) - .map_err(|err| FileError::IOError(err))?; + .map_err(|err| ReadFileError::IOError(err))?; let file_type = String::from( mime_guess::from_path(path) .first_or_octet_stream() @@ -77,16 +96,17 @@ impl FileInfo { }) } - fn hash_file(path: &Path) -> Result { + fn hash_file(path: &Path) -> Result { let mut buf = Vec::new(); - let mut file = std::fs::File::open(path).map_err(FileError::from)?; + let mut file = std::fs::File::open(path).map_err(ReadFileError::from)?; - file.read_to_end(&mut buf).map_err(FileError::from)?; + file.read_to_end(&mut buf).map_err(ReadFileError::from)?; let mut vec = Vec::new(); vec.extend_from_slice(Sha256::digest(&buf).as_slice()); Ok(HexString::from_bytes(&vec)) } + /* fn metadata_path(id: &str, root: &Path) -> PathBuf { let mut path = PathBuf::from(root); path.push(".metadata"); @@ -98,14 +118,23 @@ impl FileInfo { let path = FileInfo::metadata_path(&self.id, &self.root); remove_file(path).map_err(FileError::from) } + */ } #[cfg(test)] mod test { use super::*; - use crate::lib::utils::FileCleanup; + struct FileContext(PathBuf); + + impl FileRoot for FileContext { + fn root(&self) -> PathBuf { + self.0.clone() + } + } + + /* #[test] fn it_generates_information_from_file_path() { let path = Path::new("fixtures/rawr.png"); @@ -131,17 +160,23 @@ mod test { } } } + */ #[test] fn it_saves_and_loads_metadata() { let path = Path::new("fixtures/rawr.png"); let _ = FileCleanup(append_extension(path, "json")); let info = FileInfo::from_path(&path).unwrap(); - info.save(Path::new("fixtures")).unwrap(); + info.save(&PathResolver(PathBuf::from("fixtures/rawr.png"))) + .unwrap(); assert!(Path::new("fixtures/.metadata/rawr.png.json").is_file()); - let info_ = FileInfo::open("rawr.png", Path::new("fixtures")).unwrap(); + let info_ = FileInfo::load( + FileId::from("rawr.png"), + FileContext(PathBuf::from("fixtures")), + ) + .unwrap(); assert_eq!(info_.id, "rawr.png"); assert_eq!(info_.size, 23777); assert_eq!(info_.created, info.created); diff --git a/file-service/src/lib/mod.rs b/file-service/src/lib/mod.rs index 0d43cac..99b2f5b 100644 --- a/file-service/src/lib/mod.rs +++ b/file-service/src/lib/mod.rs @@ -1,4 +1,8 @@ -use std::path::{Path, PathBuf}; +use std::{ + ops::Deref, + path::{Path, PathBuf}, +}; +use thiserror::Error; use uuid::Uuid; mod file; @@ -6,10 +10,98 @@ mod fileinfo; mod thumbnail; pub mod utils; -pub use file::{File, FileError}; +pub use file::File; pub use fileinfo::FileInfo; pub use thumbnail::Thumbnail; +#[derive(Debug, Error)] +pub enum WriteFileError { + #[error("root file path does not exist")] + RootNotFound, + + #[error("permission denied")] + PermissionDenied, + + #[error("JSON error")] + JSONError(#[from] serde_json::error::Error), + + #[error("IO error")] + IOError(#[from] std::io::Error), +} + +#[derive(Debug, Error)] +pub enum ReadFileError { + #[error("file not found")] + FileNotFound, + + #[error("path is not a file")] + NotAFile, + + #[error("permission denied")] + PermissionDenied, + + #[error("JSON error")] + JSONError(#[from] serde_json::error::Error), + + #[error("IO error")] + IOError(#[from] std::io::Error), +} + +#[derive(Clone, Debug)] +pub struct PathResolver(pub PathBuf); + +impl PathResolver { + fn file_path(&self) -> PathBuf { + self.0.clone() + } + + fn metadata_path(&self) -> PathBuf { + let mut path = self.0.clone(); + path.set_extension("json"); + path + } + + fn thumbnail_path(&self) -> PathBuf { + let mut path = self.0.clone(); + path.set_extension("tn"); + path + } +} + +#[derive(Clone, Debug)] +pub struct FileId(String); + +impl From for FileId { + fn from(s: String) -> Self { + Self(s) + } +} + +impl From<&str> for FileId { + fn from(s: &str) -> Self { + Self(s.to_owned()) + } +} + +impl Deref for FileId { + type Target = String; + fn deref(&self) -> &Self::Target { + &self.0 + } +} + +pub trait FileRoot { + fn root(&self) -> PathBuf; +} + +pub struct Context(PathBuf); + +impl FileRoot for Context { + fn root(&self) -> PathBuf { + self.0.clone() + } +} + pub struct App { files_root: PathBuf, } @@ -21,19 +113,18 @@ impl App { } } - pub fn list_files(&self) -> Vec> { - File::list(&self.files_root) + pub fn list_files(&self) -> Vec> { + unimplemented!() } - pub fn add_file( - &mut self, - temp_path: &PathBuf, - filename: &Option, - ) -> Result { - let id = Uuid::new_v4().hyphenated().to_string(); - File::new(&id, &self.files_root, temp_path, filename) + pub fn add_file(&mut self, filename: String, content: Vec) -> Result { + let context = Context(self.files_root.clone()); + let mut file = File::new(filename, context)?; + file.set_content(content)?; + Ok(file) } + /* pub fn delete_file(&mut self, id: String) -> Result<(), FileError> { let f = File::open(&id, &self.files_root)?; f.delete() @@ -42,17 +133,24 @@ impl App { pub fn get_metadata(&self, id: String) -> Result { FileInfo::open(&id, &self.files_root) } + */ - pub fn get_file(&self, id: &str) -> Result<(FileInfo, std::fs::File), FileError> { + pub fn get_file(&self, id: &str) -> Result<(FileInfo, std::fs::File), ReadFileError> { + /* let f = File::open(&id, &self.files_root)?; let info = f.info(); let stream = f.stream()?; Ok((info, stream)) + */ + unimplemented!() } - pub fn get_thumbnail(&self, id: &str) -> Result<(FileInfo, std::fs::File), FileError> { + pub fn get_thumbnail(&self, id: &str) -> Result<(FileInfo, std::fs::File), ReadFileError> { + /* let f = File::open(id, &self.files_root)?; let stream = f.thumbnail().stream()?; Ok((f.info(), stream)) + */ + unimplemented!() } } diff --git a/file-service/src/lib/thumbnail.rs b/file-service/src/lib/thumbnail.rs index 1e458ad..5bdbd36 100644 --- a/file-service/src/lib/thumbnail.rs +++ b/file-service/src/lib/thumbnail.rs @@ -2,7 +2,7 @@ use image::imageops::FilterType; use std::fs::remove_file; use std::path::{Path, PathBuf}; -use crate::FileError; +use super::{ReadFileError, WriteFileError}; #[derive(Clone, Debug, PartialEq)] pub struct Thumbnail { @@ -11,7 +11,8 @@ pub struct Thumbnail { } impl Thumbnail { - pub fn open(id: &str, root: &Path) -> Result { + pub fn open(id: &str, root: &Path) -> Result { + /* let mut source_path = PathBuf::from(root); source_path.push(id); @@ -28,20 +29,24 @@ impl Thumbnail { } Ok(self_) + */ + unimplemented!() } - pub fn from_path(path: &Path) -> Result { + /* + pub fn from_path(path: &Path) -> Result { let id = path .file_name() .map(|s| String::from(s.to_string_lossy())) - .ok_or(FileError::NotAnImage(PathBuf::from(path)))?; + .ok_or(ReadFileError::NotAnImage(PathBuf::from(path)))?; let root = path .parent() - .ok_or(FileError::FileNotFound(PathBuf::from(path)))?; + .ok_or(ReadFileError::FileNotFound(PathBuf::from(path)))?; Thumbnail::open(&id, root) } + */ fn thumbnail_path(id: &str, root: &Path) -> PathBuf { let mut path = PathBuf::from(root); @@ -50,20 +55,20 @@ impl Thumbnail { path } - pub fn stream(&self) -> Result { + pub fn stream(&self) -> Result { let thumbnail_path = Thumbnail::thumbnail_path(&self.id, &self.root); std::fs::File::open(thumbnail_path.clone()).map_err(|err| { if err.kind() == std::io::ErrorKind::NotFound { - FileError::FileNotFound(thumbnail_path) + ReadFileError::FileNotFound } else { - FileError::from(err) + ReadFileError::from(err) } }) } - pub fn delete(&self) -> Result<(), FileError> { + pub fn delete(&self) -> Result<(), WriteFileError> { let path = Thumbnail::thumbnail_path(&self.id, &self.root); - remove_file(path).map_err(FileError::from) + remove_file(path).map_err(WriteFileError::from) } } diff --git a/file-service/src/main.rs b/file-service/src/main.rs index efb3dbb..11ba5ec 100644 --- a/file-service/src/main.rs +++ b/file-service/src/main.rs @@ -13,6 +13,8 @@ use http::status::StatusCode; // use mustache::{compile_path, Template}; // use orizentic::{Permissions, ResourceName, Secret}; use build_html::Html; +use bytes::Buf; +use futures_util::{Stream, StreamExt}; use std::{ collections::HashMap, io::Read, @@ -20,7 +22,7 @@ use std::{ path::Path, sync::{Arc, RwLock}, }; -use warp::Filter; +use warp::{filters::multipart::Part, Filter}; mod cookies; mod html; @@ -28,7 +30,7 @@ mod lib; mod middleware; mod pages; -use lib::{utils::append_extension, App, File, FileError, FileInfo}; +use lib::{utils::append_extension, App, File, FileInfo}; /* fn is_admin(resource: &ResourceName, permissions: &Permissions) -> bool { @@ -247,6 +249,32 @@ fn serve_file( } } +async fn collect_content(mut part: Part) -> Result<(Option, Vec), String> { + let mut content: Vec = Vec::new(); + + while let Some(Ok(data)) = part.data().await { + let mut reader = data.reader(); + reader.read_to_end(&mut content).unwrap(); + } + + Ok((part.filename().map(|s| s.to_owned()), content)) +} + +async fn collect_multipart( + mut stream: warp::filters::multipart::FormData, +) -> Result, Vec)>, warp::Error> { + let mut content: Vec<(Option, Vec)> = Vec::new(); + + while let Some(part) = stream.next().await { + match part { + Ok(part) => content.push(collect_content(part).await.unwrap()), + Err(err) => return Err(err), + } + } + + Ok(content) +} + #[tokio::main] pub async fn main() { /* @@ -309,6 +337,7 @@ pub async fn main() { } }); + /* let post_handler = warp::path!(String) .and(warp::post()) .and(warp::filters::body::form()) @@ -320,6 +349,7 @@ pub async fn main() { .status(StatusCode::SEE_OTHER) .body(vec![]) }); + */ let thumbnail = warp::path!(String / "tn") .and(warp::get()) @@ -351,16 +381,29 @@ pub async fn main() { } }); - let upload = warp::path!().and(warp::post()).map(|| { - println!("upload"); - warp::reply() - }); + let upload = warp::path!() + .and(warp::post()) + .and(warp::filters::multipart::form().max_length(1024 * 1024 * 32)) + .then(|form: warp::filters::multipart::FormData| async move { + let files = collect_multipart(form).await; + /* + for (filename, content) in files { + app.write() + .unwrap() + .add_file(Some(filename), content) + .unwrap(); + } + */ + println!("file length: {:?}", files.map(|f| f.len())); + warp::reply() + }); let delete = warp::path!(String).and(warp::delete()).map(|id: String| { println!("delete {}", id); warp::reply() }); + /* let server = warp::serve( root.or(post_handler) .or(file) @@ -369,6 +412,8 @@ pub async fn main() { .or(delete) .with(log), ); + */ + let server = warp::serve(root.or(upload).with(log)); server .run(SocketAddr::new(IpAddr::V4(Ipv4Addr::new(0, 0, 0, 0)), 8002)) .await; diff --git a/file-service/src/pages.rs b/file-service/src/pages.rs index 790b644..7058f65 100644 --- a/file-service/src/pages.rs +++ b/file-service/src/pages.rs @@ -1,7 +1,7 @@ -use crate::{html::*, File, FileError}; +use crate::{html::*, lib::ReadFileError, File}; use build_html::{self, Container, ContainerType, Html, HtmlContainer}; -pub fn index(files: Vec>) -> build_html::HtmlPage { +pub fn index(files: Vec>) -> build_html::HtmlPage { let mut page = build_html::HtmlPage::new() .with_title("Admin list of files") .with_header(1, "Admin list of files") @@ -21,7 +21,7 @@ pub fn index(files: Vec>) -> build_html::HtmlPage { let container = match file { Ok(ref file) => thumbnail(file).with_html( Form::new() - .with_path(&format!("/{}", file.info().id)) + .with_path(&format!("/{}", *file.id)) .with_method("post") .with_html(Input::new("hidden", "_method").with_value("delete")) .with_html(Button::new("Delete")), @@ -41,8 +41,8 @@ pub fn thumbnail(file: &File) -> Container { let tn = Container::new(ContainerType::Div) .with_attributes(vec![("class", "thumbnail")]) .with_link( - format!("/{}", file.info().id), - Image::new(&format!("{}/tn", file.info().id)).to_html_string(), + format!("/{}", *file.id), + Image::new(&format!("{}/tn", *file.id)).to_html_string(), ); container.add_html(tn); container