Start ripping out lots of infrastructure

Much of the infrastructure is old and seems to be based on some assumptions about how Iron handled multipart posts. I don't understand how much of this works, so I'm slowly ripping parts out and rebuilding how the separation of concerns works.

Cargo.lock

@@ -703,7 +703,9 @@ name = "file-service"
 version = "0.1.0"
 dependencies = [
  "build_html",
+ "bytes",
  "chrono",
+ "futures-util",
  "hex-string",
  "http",
  "image 0.23.14",

file-service/.gitignore

@@ -0,0 +1 @@
+fixtures

file-service/.ignore

@@ -0,0 +1 @@
+fixtures

file-service/Cargo.toml

@@ -29,3 +29,6 @@ uuid        = { version = "0.4", features = [ "serde", "v4" ] }
 warp        = { version = "0.3" }
 pretty_env_logger = { version = "0.5" }
 log         = { version = "0.4" }
+bytes       = { version = "1" }
+futures-util    = { version = "0.3" }
+

file-service/src/lib/file.rs

@@ -1,9 +1,17 @@
-use super::fileinfo::FileInfo;
+use super::{
-use super::thumbnail::Thumbnail;
+    fileinfo::FileInfo, thumbnail::Thumbnail, FileId, FileRoot, PathResolver, ReadFileError,
-use std::fs::{copy, read_dir, remove_file};
+    WriteFileError,
-use std::path::{Path, PathBuf};
+};
+use chrono::prelude::*;
+use std::{
+    fs::{copy, read_dir, remove_file},
+    io::{Read, Write},
+    path::{Path, PathBuf},
+};
 use thiserror::Error;
+use uuid::Uuid;
 
+/*
 #[derive(Error, Debug)]
 pub enum FileError {
     #[error("not implemented")]
@@ -30,21 +38,79 @@ pub enum FileError {
     #[error("UTF8 Error")]
     UTF8Error(#[from] std::str::Utf8Error),
 }
+*/
 
 /// One file in the database, complete with the path of the file and information about the
 /// thumbnail of the file.
 #[derive(Debug)]
 pub struct File {
-    info: FileInfo,
+    pub id: FileId,
-    tn: Thumbnail,
+    path: PathResolver,
-    root: PathBuf,
+    pub info: FileInfo,
+    tn: Option<Thumbnail>,
 }
 
 impl File {
+    /// Create a new entry in the database
+    pub fn new<CTX: FileRoot>(filename: String, context: CTX) -> Result<Self, WriteFileError> {
+        let id = FileId::from(Uuid::new_v4().hyphenated().to_string());
+
+        let mut path = context.root();
+        path.push((*id).to_owned());
+        let path = PathResolver(path);
+
+        let file_type = mime_guess::from_ext(&filename)
+            .first_or_text_plain()
+            .essence_str()
+            .to_owned();
+
+        let info = FileInfo {
+            id: (*id).to_owned(),
+            size: 0,
+            created: Utc::now(),
+            file_type,
+            hash: "".to_owned(),
+            root: context.root().to_owned(),
+        };
+
+        let mut md_file = std::fs::File::create(path.metadata_path())?;
+        md_file.write(&serde_json::to_vec(&info)?)?;
+
+        Ok(Self {
+            id,
+            path,
+            info,
+            tn: None,
+        })
+    }
+
+    pub fn load<CTX: FileRoot>(id: FileId, context: CTX) -> Result<Self, ReadFileError> {
+        let mut path = context.root();
+        path.push((*id).to_owned());
+        let path = PathResolver(path);
+
+        Ok(Self {
+            id: id.clone(),
+            path,
+            info: FileInfo::load(id, context)?,
+            tn: None,
+        })
+    }
+
+    pub fn set_content(&self, content: Vec<u8>) -> Result<(), WriteFileError> {
+        let mut content_file = std::fs::File::create(self.path.file_path())?;
+        content_file.write(&content)?;
+        Ok(())
+    }
+
+    pub fn content(&self) -> Result<Vec<u8>, ReadFileError> {
+        unimplemented!()
+    }
+
+    /*
     pub fn new(
         id: &str,
         root: &Path,
-        temp_path: &PathBuf,
         filename: &Option<PathBuf>,
     ) -> Result<Self, FileError> {
         let mut dest_path = PathBuf::from(root);
@@ -136,6 +202,7 @@ impl File {
         self.tn.delete()?;
         self.info.delete()
     }
+    */
 }
 
 #[cfg(test)]
@@ -144,17 +211,34 @@ mod test {
     use crate::lib::utils::FileCleanup;
     use std::path::{Path, PathBuf};
 
+    struct FileContext(PathBuf);
+
+    impl FileRoot for FileContext {
+        fn root(&self) -> PathBuf {
+            self.0.clone()
+        }
+    }
+
     #[test]
     fn it_opens_a_file() {
         let _md = FileCleanup(PathBuf::from("fixtures/.metadata/rawr.png.json"));
         let _tn = FileCleanup(PathBuf::from("fixtures/.thumbnails/rawr.png"));
 
-        File::open("rawr.png", Path::new("fixtures/")).expect("to succeed");
+        File::new(
+            "rawr.png".to_owned(),
+            FileContext(PathBuf::from("fixtures/")),
+        )
+        .expect("to succeed");
     }
 
     #[test]
     fn it_can_return_a_thumbnail() {
-        let f = File::open("rawr.png", Path::new("fixtures/")).expect("to succeed");
+        let f = File::new(
+            "rawr.png".to_owned(),
+            FileContext(PathBuf::from("fixtures/")),
+        )
+        .expect("to succeed");
+        /*
         assert_eq!(
             f.thumbnail(),
             Thumbnail {
@@ -162,18 +246,26 @@ mod test {
                 root: PathBuf::from("fixtures/"),
             },
         );
+        */
     }
 
     #[test]
     fn it_can_return_a_file_stream() {
-        let f = File::open("rawr.png", Path::new("fixtures/")).expect("to succeed");
+        let f = File::new(
-        f.stream().expect("to succeed");
+            "rawr.png".to_owned(),
+            FileContext(PathBuf::from("fixtures/")),
+        )
+        .expect("to succeed");
+        // f.stream().expect("to succeed");
     }
 
     #[test]
     fn it_raises_an_error_when_file_not_found() {
-        match File::open("garbage", Path::new("fixtures/")) {
+        match File::load(
-            Err(Error::FileNotFound(_)) => assert!(true),
+            FileId::from("rawr.png"),
+            FileContext(PathBuf::from("fixtures/")),
+        ) {
+            Err(ReadFileError::FileNotFound) => assert!(true),
             _ => assert!(false),
         }
     }

file-service/src/lib/fileinfo.rs

@@ -7,7 +7,8 @@ use std::fs::remove_file;
 use std::io::{Read, Write};
 use std::path::{Path, PathBuf};
 
-use crate::{append_extension, FileError};
+use super::{FileId, FileRoot, PathResolver, ReadFileError};
+use crate::append_extension;
 
 #[derive(Clone, Debug, Serialize, Deserialize)]
 pub struct FileInfo {
@@ -18,17 +19,34 @@ pub struct FileInfo {
     pub hash: String,
 
     #[serde(skip)]
-    root: PathBuf,
+    pub root: PathBuf,
 }
 
 impl FileInfo {
-    pub fn save(&self, root: &Path) -> Result<(), FileError> {
+    pub fn load<CTX: FileRoot>(id: FileId, context: CTX) -> Result<Self, ReadFileError> {
-        let ser = serde_json::to_string(self).unwrap();
+        let mut content: Vec<u8> = Vec::new();
-        std::fs::File::create(FileInfo::metadata_path(&self.id, root))
+        let mut file = std::fs::File::open(Self::path(id, context))?;
-            .and_then(|mut stream| stream.write(ser.as_bytes()).map(|_| (())))
+        file.read_to_end(&mut content)?;
-            .map_err(FileError::from)
+        let js = serde_json::from_slice(&content)?;
+
+        Ok(js)
     }
 
+    pub fn path<CTX: FileRoot>(id: FileId, context: CTX) -> PathBuf {
+        let mut path = context.root();
+        path.push((*id).to_owned());
+        path.set_extension("json");
+        path
+    }
+
+    pub fn save(&self, root: &PathResolver) -> Result<(), ReadFileError> {
+        let ser = serde_json::to_string(self).unwrap();
+        std::fs::File::create(root.metadata_path())
+            .and_then(|mut stream| stream.write(ser.as_bytes()).map(|_| (())))
+            .map_err(ReadFileError::from)
+    }
+
+    /*
     pub fn open(id: &str, root: &Path) -> Result<FileInfo, FileError> {
         let mut buf = Vec::new();
 
@@ -44,23 +62,24 @@ impl FileInfo {
 
         serde_json::from_str(&str_repr).map_err(FileError::from)
     }
+    */
 
-    pub fn from_path(path: &Path) -> Result<FileInfo, FileError> {
+    pub fn from_path(path: &Path) -> Result<FileInfo, ReadFileError> {
         match (path.is_file(), path.is_dir()) {
-            (false, false) => Err(FileError::FileNotFound(PathBuf::from(path))),
+            (false, false) => Err(ReadFileError::FileNotFound),
-            (false, true) => Err(FileError::NotAFile(PathBuf::from(path))),
+            (false, true) => Err(ReadFileError::NotAFile),
             (true, _) => Ok(()),
         }?;
 
-        let metadata = path.metadata().map_err(FileError::IOError)?;
+        let metadata = path.metadata().map_err(ReadFileError::IOError)?;
         let id = path
             .file_name()
             .map(|s| String::from(s.to_string_lossy()))
-            .ok_or(FileError::NotAFile(PathBuf::from(path)))?;
+            .ok_or(ReadFileError::NotAFile)?;
         let created = metadata
             .created()
             .map(|m| DateTime::from(m))
-            .map_err(|err| FileError::IOError(err))?;
+            .map_err(|err| ReadFileError::IOError(err))?;
         let file_type = String::from(
             mime_guess::from_path(path)
                 .first_or_octet_stream()
@@ -77,16 +96,17 @@ impl FileInfo {
         })
     }
 
-    fn hash_file(path: &Path) -> Result<HexString, FileError> {
+    fn hash_file(path: &Path) -> Result<HexString, ReadFileError> {
         let mut buf = Vec::new();
-        let mut file = std::fs::File::open(path).map_err(FileError::from)?;
+        let mut file = std::fs::File::open(path).map_err(ReadFileError::from)?;
 
-        file.read_to_end(&mut buf).map_err(FileError::from)?;
+        file.read_to_end(&mut buf).map_err(ReadFileError::from)?;
         let mut vec = Vec::new();
         vec.extend_from_slice(Sha256::digest(&buf).as_slice());
         Ok(HexString::from_bytes(&vec))
     }
 
+    /*
     fn metadata_path(id: &str, root: &Path) -> PathBuf {
         let mut path = PathBuf::from(root);
         path.push(".metadata");
@@ -98,14 +118,23 @@ impl FileInfo {
         let path = FileInfo::metadata_path(&self.id, &self.root);
         remove_file(path).map_err(FileError::from)
     }
+    */
 }
 
 #[cfg(test)]
 mod test {
     use super::*;
-
     use crate::lib::utils::FileCleanup;
 
+    struct FileContext(PathBuf);
+
+    impl FileRoot for FileContext {
+        fn root(&self) -> PathBuf {
+            self.0.clone()
+        }
+    }
+
+    /*
     #[test]
     fn it_generates_information_from_file_path() {
         let path = Path::new("fixtures/rawr.png");
@@ -131,17 +160,23 @@ mod test {
             }
         }
     }
+    */
 
     #[test]
     fn it_saves_and_loads_metadata() {
         let path = Path::new("fixtures/rawr.png");
         let _ = FileCleanup(append_extension(path, "json"));
         let info = FileInfo::from_path(&path).unwrap();
-        info.save(Path::new("fixtures")).unwrap();
+        info.save(&PathResolver(PathBuf::from("fixtures/rawr.png")))
+            .unwrap();
 
         assert!(Path::new("fixtures/.metadata/rawr.png.json").is_file());
 
-        let info_ = FileInfo::open("rawr.png", Path::new("fixtures")).unwrap();
+        let info_ = FileInfo::load(
+            FileId::from("rawr.png"),
+            FileContext(PathBuf::from("fixtures")),
+        )
+        .unwrap();
         assert_eq!(info_.id, "rawr.png");
         assert_eq!(info_.size, 23777);
         assert_eq!(info_.created, info.created);

file-service/src/lib/mod.rs

@@ -1,4 +1,8 @@
-use std::path::{Path, PathBuf};
+use std::{
+    ops::Deref,
+    path::{Path, PathBuf},
+};
+use thiserror::Error;
 use uuid::Uuid;
 
 mod file;
@@ -6,10 +10,98 @@ mod fileinfo;
 mod thumbnail;
 pub mod utils;
 
-pub use file::{File, FileError};
+pub use file::File;
 pub use fileinfo::FileInfo;
 pub use thumbnail::Thumbnail;
 
+#[derive(Debug, Error)]
+pub enum WriteFileError {
+    #[error("root file path does not exist")]
+    RootNotFound,
+
+    #[error("permission denied")]
+    PermissionDenied,
+
+    #[error("JSON error")]
+    JSONError(#[from] serde_json::error::Error),
+
+    #[error("IO error")]
+    IOError(#[from] std::io::Error),
+}
+
+#[derive(Debug, Error)]
+pub enum ReadFileError {
+    #[error("file not found")]
+    FileNotFound,
+
+    #[error("path is not a file")]
+    NotAFile,
+
+    #[error("permission denied")]
+    PermissionDenied,
+
+    #[error("JSON error")]
+    JSONError(#[from] serde_json::error::Error),
+
+    #[error("IO error")]
+    IOError(#[from] std::io::Error),
+}
+
+#[derive(Clone, Debug)]
+pub struct PathResolver(pub PathBuf);
+
+impl PathResolver {
+    fn file_path(&self) -> PathBuf {
+        self.0.clone()
+    }
+
+    fn metadata_path(&self) -> PathBuf {
+        let mut path = self.0.clone();
+        path.set_extension("json");
+        path
+    }
+
+    fn thumbnail_path(&self) -> PathBuf {
+        let mut path = self.0.clone();
+        path.set_extension("tn");
+        path
+    }
+}
+
+#[derive(Clone, Debug)]
+pub struct FileId(String);
+
+impl From<String> for FileId {
+    fn from(s: String) -> Self {
+        Self(s)
+    }
+}
+
+impl From<&str> for FileId {
+    fn from(s: &str) -> Self {
+        Self(s.to_owned())
+    }
+}
+
+impl Deref for FileId {
+    type Target = String;
+    fn deref(&self) -> &Self::Target {
+        &self.0
+    }
+}
+
+pub trait FileRoot {
+    fn root(&self) -> PathBuf;
+}
+
+pub struct Context(PathBuf);
+
+impl FileRoot for Context {
+    fn root(&self) -> PathBuf {
+        self.0.clone()
+    }
+}
+
 pub struct App {
     files_root: PathBuf,
 }
@@ -21,19 +113,18 @@ impl App {
         }
     }
 
-    pub fn list_files(&self) -> Vec<Result<File, FileError>> {
+    pub fn list_files(&self) -> Vec<Result<File, ReadFileError>> {
-        File::list(&self.files_root)
+        unimplemented!()
     }
 
-    pub fn add_file(
+    pub fn add_file(&mut self, filename: String, content: Vec<u8>) -> Result<File, WriteFileError> {
-        &mut self,
+        let context = Context(self.files_root.clone());
-        temp_path: &PathBuf,
+        let mut file = File::new(filename, context)?;
-        filename: &Option<PathBuf>,
+        file.set_content(content)?;
-    ) -> Result<File, FileError> {
+        Ok(file)
-        let id = Uuid::new_v4().hyphenated().to_string();
-        File::new(&id, &self.files_root, temp_path, filename)
     }
 
+    /*
     pub fn delete_file(&mut self, id: String) -> Result<(), FileError> {
         let f = File::open(&id, &self.files_root)?;
         f.delete()
@@ -42,17 +133,24 @@ impl App {
     pub fn get_metadata(&self, id: String) -> Result<FileInfo, FileError> {
         FileInfo::open(&id, &self.files_root)
     }
+    */
 
-    pub fn get_file(&self, id: &str) -> Result<(FileInfo, std::fs::File), FileError> {
+    pub fn get_file(&self, id: &str) -> Result<(FileInfo, std::fs::File), ReadFileError> {
+        /*
         let f = File::open(&id, &self.files_root)?;
         let info = f.info();
         let stream = f.stream()?;
         Ok((info, stream))
+        */
+        unimplemented!()
     }
 
-    pub fn get_thumbnail(&self, id: &str) -> Result<(FileInfo, std::fs::File), FileError> {
+    pub fn get_thumbnail(&self, id: &str) -> Result<(FileInfo, std::fs::File), ReadFileError> {
+        /*
         let f = File::open(id, &self.files_root)?;
         let stream = f.thumbnail().stream()?;
         Ok((f.info(), stream))
+        */
+        unimplemented!()
     }
 }

file-service/src/lib/thumbnail.rs

@@ -2,7 +2,7 @@ use image::imageops::FilterType;
 use std::fs::remove_file;
 use std::path::{Path, PathBuf};
 
-use crate::FileError;
+use super::{ReadFileError, WriteFileError};
 
 #[derive(Clone, Debug, PartialEq)]
 pub struct Thumbnail {
@@ -11,7 +11,8 @@ pub struct Thumbnail {
 }
 
 impl Thumbnail {
-    pub fn open(id: &str, root: &Path) -> Result<Thumbnail, FileError> {
+    pub fn open(id: &str, root: &Path) -> Result<Thumbnail, WriteFileError> {
+        /*
         let mut source_path = PathBuf::from(root);
         source_path.push(id);
 
@@ -28,20 +29,24 @@ impl Thumbnail {
         }
 
         Ok(self_)
+        */
+        unimplemented!()
     }
 
-    pub fn from_path(path: &Path) -> Result<Thumbnail, FileError> {
+    /*
+    pub fn from_path(path: &Path) -> Result<Thumbnail, ReadFileError> {
         let id = path
             .file_name()
             .map(|s| String::from(s.to_string_lossy()))
-            .ok_or(FileError::NotAnImage(PathBuf::from(path)))?;
+            .ok_or(ReadFileError::NotAnImage(PathBuf::from(path)))?;
 
         let root = path
             .parent()
-            .ok_or(FileError::FileNotFound(PathBuf::from(path)))?;
+            .ok_or(ReadFileError::FileNotFound(PathBuf::from(path)))?;
 
         Thumbnail::open(&id, root)
     }
+    */
 
     fn thumbnail_path(id: &str, root: &Path) -> PathBuf {
         let mut path = PathBuf::from(root);
@@ -50,20 +55,20 @@ impl Thumbnail {
         path
     }
 
-    pub fn stream(&self) -> Result<std::fs::File, FileError> {
+    pub fn stream(&self) -> Result<std::fs::File, ReadFileError> {
         let thumbnail_path = Thumbnail::thumbnail_path(&self.id, &self.root);
         std::fs::File::open(thumbnail_path.clone()).map_err(|err| {
             if err.kind() == std::io::ErrorKind::NotFound {
-                FileError::FileNotFound(thumbnail_path)
+                ReadFileError::FileNotFound
             } else {
-                FileError::from(err)
+                ReadFileError::from(err)
             }
         })
     }
 
-    pub fn delete(&self) -> Result<(), FileError> {
+    pub fn delete(&self) -> Result<(), WriteFileError> {
         let path = Thumbnail::thumbnail_path(&self.id, &self.root);
-        remove_file(path).map_err(FileError::from)
+        remove_file(path).map_err(WriteFileError::from)
     }
 }
 

file-service/src/main.rs

@@ -13,6 +13,8 @@ use http::status::StatusCode;
 // use mustache::{compile_path, Template};
 // use orizentic::{Permissions, ResourceName, Secret};
 use build_html::Html;
+use bytes::Buf;
+use futures_util::{Stream, StreamExt};
 use std::{
     collections::HashMap,
     io::Read,
@@ -20,7 +22,7 @@ use std::{
     path::Path,
     sync::{Arc, RwLock},
 };
-use warp::Filter;
+use warp::{filters::multipart::Part, Filter};
 
 mod cookies;
 mod html;
@@ -28,7 +30,7 @@ mod lib;
 mod middleware;
 mod pages;
 
-use lib::{utils::append_extension, App, File, FileError, FileInfo};
+use lib::{utils::append_extension, App, File, FileInfo};
 
 /*
 fn is_admin(resource: &ResourceName, permissions: &Permissions) -> bool {
@@ -247,6 +249,32 @@ fn serve_file(
     }
 }
 
+async fn collect_content(mut part: Part) -> Result<(Option<String>, Vec<u8>), String> {
+    let mut content: Vec<u8> = Vec::new();
+
+    while let Some(Ok(data)) = part.data().await {
+        let mut reader = data.reader();
+        reader.read_to_end(&mut content).unwrap();
+    }
+
+    Ok((part.filename().map(|s| s.to_owned()), content))
+}
+
+async fn collect_multipart(
+    mut stream: warp::filters::multipart::FormData,
+) -> Result<Vec<(Option<String>, Vec<u8>)>, warp::Error> {
+    let mut content: Vec<(Option<String>, Vec<u8>)> = Vec::new();
+
+    while let Some(part) = stream.next().await {
+        match part {
+            Ok(part) => content.push(collect_content(part).await.unwrap()),
+            Err(err) => return Err(err),
+        }
+    }
+
+    Ok(content)
+}
+
 #[tokio::main]
 pub async fn main() {
     /*
@@ -309,6 +337,7 @@ pub async fn main() {
         }
     });
 
+    /*
     let post_handler = warp::path!(String)
         .and(warp::post())
         .and(warp::filters::body::form())
@@ -320,6 +349,7 @@ pub async fn main() {
                 .status(StatusCode::SEE_OTHER)
                 .body(vec![])
         });
+        */
 
     let thumbnail = warp::path!(String / "tn")
         .and(warp::get())
@@ -351,16 +381,29 @@ pub async fn main() {
             }
         });
 
-    let upload = warp::path!().and(warp::post()).map(|| {
+    let upload = warp::path!()
-        println!("upload");
+        .and(warp::post())
-        warp::reply()
+        .and(warp::filters::multipart::form().max_length(1024 * 1024 * 32))
-    });
+        .then(|form: warp::filters::multipart::FormData| async move {
+            let files = collect_multipart(form).await;
+            /*
+            for (filename, content) in files {
+                app.write()
+                    .unwrap()
+                    .add_file(Some(filename), content)
+                    .unwrap();
+            }
+            */
+            println!("file length: {:?}", files.map(|f| f.len()));
+            warp::reply()
+        });
 
     let delete = warp::path!(String).and(warp::delete()).map(|id: String| {
         println!("delete {}", id);
         warp::reply()
     });
 
+    /*
     let server = warp::serve(
         root.or(post_handler)
             .or(file)
@@ -369,6 +412,8 @@ pub async fn main() {
             .or(delete)
             .with(log),
     );
+    */
+    let server = warp::serve(root.or(upload).with(log));
     server
         .run(SocketAddr::new(IpAddr::V4(Ipv4Addr::new(0, 0, 0, 0)), 8002))
         .await;

file-service/src/pages.rs

@@ -1,7 +1,7 @@
-use crate::{html::*, File, FileError};
+use crate::{html::*, lib::ReadFileError, File};
 use build_html::{self, Container, ContainerType, Html, HtmlContainer};
 
-pub fn index(files: Vec<Result<File, FileError>>) -> build_html::HtmlPage {
+pub fn index(files: Vec<Result<File, ReadFileError>>) -> build_html::HtmlPage {
     let mut page = build_html::HtmlPage::new()
         .with_title("Admin list of files")
         .with_header(1, "Admin list of files")
@@ -21,7 +21,7 @@ pub fn index(files: Vec<Result<File, FileError>>) -> build_html::HtmlPage {
         let container = match file {
             Ok(ref file) => thumbnail(file).with_html(
                 Form::new()
-                    .with_path(&format!("/{}", file.info().id))
+                    .with_path(&format!("/{}", *file.id))
                     .with_method("post")
                     .with_html(Input::new("hidden", "_method").with_value("delete"))
                     .with_html(Button::new("Delete")),
@@ -41,8 +41,8 @@ pub fn thumbnail(file: &File) -> Container {
     let tn = Container::new(ContainerType::Div)
         .with_attributes(vec![("class", "thumbnail")])
         .with_link(
-            format!("/{}", file.info().id),
+            format!("/{}", *file.id),
-            Image::new(&format!("{}/tn", file.info().id)).to_html_string(),
+            Image::new(&format!("{}/tn", *file.id)).to_html_string(),
         );
     container.add_html(tn);
     container