savanni
/
monorepo
1
0
Fork 0
Code Issues 76 Pull Requests Projects 6 Releases Wiki Activity

Start ripping out lots of infrastructure 

Much of the infrastructure is old and seems to be based on some assumptions about how Iron handled multipart posts. I don't understand how much of this works, so I'm slowly ripping parts out and rebuilding how the separation of concerns works.
This commit is contained in:
Savanni D'Gerinel 2023-09-22 20:03:58 -04:00
parent 8521db333b
commit 396f6e3bcf
10 changed files with 349 additions and 67 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
Cargo.lock generated
View File

@ -703,7 +703,9 @@ name = "file-service"
version = "0.1.0"
dependencies = [
"build_html",
"bytes",
"chrono",
"futures-util",
"hex-string",
"http",
"image 0.23.14",

1
file-service/.gitignore vendored Normal file
View File

@ -0,0 +1 @@
fixtures

1
file-service/.ignore Normal file
View File

@ -0,0 +1 @@
fixtures

3
file-service/Cargo.toml
View File

@ -29,3 +29,6 @@ uuid = { version = "0.4", features = [ "serde", "v4" ] }
warp = { version = "0.3" }
pretty_env_logger = { version = "0.5" }
log = { version = "0.4" }
bytes = { version = "1" }
futures-util = { version = "0.3" }

120
file-service/src/lib/file.rs
View File

@ -1,9 +1,17 @@
use super::fileinfo::FileInfo;
use super::thumbnail::Thumbnail;
use std::fs::{copy, read_dir, remove_file};
use std::path::{Path, PathBuf};
use super::{
fileinfo::FileInfo, thumbnail::Thumbnail, FileId, FileRoot, PathResolver, ReadFileError,
WriteFileError,
};
use chrono::prelude::*;
use std::{
fs::{copy, read_dir, remove_file},
io::{Read, Write},
path::{Path, PathBuf},
};
use thiserror::Error;
use uuid::Uuid;
/*
#[derive(Error, Debug)]
pub enum FileError {
#[error("not implemented")]
@ -30,21 +38,79 @@ pub enum FileError {
#[error("UTF8 Error")]
UTF8Error(#[from] std::str::Utf8Error),
}
*/
/// One file in the database, complete with the path of the file and information about the
/// thumbnail of the file.
#[derive(Debug)]
pub struct File {
info: FileInfo,
tn: Thumbnail,
root: PathBuf,
pub id: FileId,
path: PathResolver,
pub info: FileInfo,
tn: Option<Thumbnail>,
}
impl File {
/// Create a new entry in the database
pub fn new<CTX: FileRoot>(filename: String, context: CTX) -> Result<Self, WriteFileError> {
let id = FileId::from(Uuid::new_v4().hyphenated().to_string());
let mut path = context.root();
path.push((*id).to_owned());
let path = PathResolver(path);
let file_type = mime_guess::from_ext(&filename)
.first_or_text_plain()
.essence_str()
.to_owned();
let info = FileInfo {
id: (*id).to_owned(),
size: 0,
created: Utc::now(),
file_type,
hash: "".to_owned(),
root: context.root().to_owned(),
};
let mut md_file = std::fs::File::create(path.metadata_path())?;
md_file.write(&serde_json::to_vec(&info)?)?;
Ok(Self {
id,
path,
info,
tn: None,
})
}
pub fn load<CTX: FileRoot>(id: FileId, context: CTX) -> Result<Self, ReadFileError> {
let mut path = context.root();
path.push((*id).to_owned());
let path = PathResolver(path);
Ok(Self {
id: id.clone(),
path,
info: FileInfo::load(id, context)?,
tn: None,
})
}
pub fn set_content(&self, content: Vec<u8>) -> Result<(), WriteFileError> {
let mut content_file = std::fs::File::create(self.path.file_path())?;
content_file.write(&content)?;
Ok(())
}
pub fn content(&self) -> Result<Vec<u8>, ReadFileError> {
unimplemented!()
}
/*
pub fn new(
id: &str,
root: &Path,
temp_path: &PathBuf,
filename: &Option<PathBuf>,
) -> Result<Self, FileError> {
let mut dest_path = PathBuf::from(root);
@ -136,6 +202,7 @@ impl File {
self.tn.delete()?;
self.info.delete()
}
*/
}
#[cfg(test)]
@ -144,17 +211,34 @@ mod test {
use crate::lib::utils::FileCleanup;
use std::path::{Path, PathBuf};
struct FileContext(PathBuf);
impl FileRoot for FileContext {
fn root(&self) -> PathBuf {
self.0.clone()
}
}
#[test]
fn it_opens_a_file() {
let _md = FileCleanup(PathBuf::from("fixtures/.metadata/rawr.png.json"));
let _tn = FileCleanup(PathBuf::from("fixtures/.thumbnails/rawr.png"));
File::open("rawr.png", Path::new("fixtures/")).expect("to succeed");
File::new(
"rawr.png".to_owned(),
FileContext(PathBuf::from("fixtures/")),
)
.expect("to succeed");
}
#[test]
fn it_can_return_a_thumbnail() {
let f = File::open("rawr.png", Path::new("fixtures/")).expect("to succeed");
let f = File::new(
"rawr.png".to_owned(),
FileContext(PathBuf::from("fixtures/")),
)
.expect("to succeed");
/*
assert_eq!(
f.thumbnail(),
Thumbnail {
@ -162,18 +246,26 @@ mod test {
root: PathBuf::from("fixtures/"),
},
);
*/
}
#[test]
fn it_can_return_a_file_stream() {
let f = File::open("rawr.png", Path::new("fixtures/")).expect("to succeed");
f.stream().expect("to succeed");
let f = File::new(
"rawr.png".to_owned(),
FileContext(PathBuf::from("fixtures/")),
)
.expect("to succeed");
// f.stream().expect("to succeed");
}
#[test]
fn it_raises_an_error_when_file_not_found() {
match File::open("garbage", Path::new("fixtures/")) {
Err(Error::FileNotFound(_)) => assert!(true),
match File::load(
FileId::from("rawr.png"),
FileContext(PathBuf::from("fixtures/")),
) {
Err(ReadFileError::FileNotFound) => assert!(true),
_ => assert!(false),
}
}

73
file-service/src/lib/fileinfo.rs
View File

@ -7,7 +7,8 @@ use std::fs::remove_file;
use std::io::{Read, Write};
use std::path::{Path, PathBuf};
use crate::{append_extension, FileError};
use super::{FileId, FileRoot, PathResolver, ReadFileError};
use crate::append_extension;
#[derive(Clone, Debug, Serialize, Deserialize)]
pub struct FileInfo {
@ -18,17 +19,34 @@ pub struct FileInfo {
pub hash: String,
#[serde(skip)]
root: PathBuf,
pub root: PathBuf,
}
impl FileInfo {
pub fn save(&self, root: &Path) -> Result<(), FileError> {
let ser = serde_json::to_string(self).unwrap();
std::fs::File::create(FileInfo::metadata_path(&self.id, root))
.and_then(|mut stream| stream.write(ser.as_bytes()).map(|_| (())))
.map_err(FileError::from)
pub fn load<CTX: FileRoot>(id: FileId, context: CTX) -> Result<Self, ReadFileError> {
let mut content: Vec<u8> = Vec::new();
let mut file = std::fs::File::open(Self::path(id, context))?;
file.read_to_end(&mut content)?;
let js = serde_json::from_slice(&content)?;
Ok(js)
}
pub fn path<CTX: FileRoot>(id: FileId, context: CTX) -> PathBuf {
let mut path = context.root();
path.push((*id).to_owned());
path.set_extension("json");
path
}
pub fn save(&self, root: &PathResolver) -> Result<(), ReadFileError> {
let ser = serde_json::to_string(self).unwrap();
std::fs::File::create(root.metadata_path())
.and_then(|mut stream| stream.write(ser.as_bytes()).map(|_| (())))
.map_err(ReadFileError::from)
}
/*
pub fn open(id: &str, root: &Path) -> Result<FileInfo, FileError> {
let mut buf = Vec::new();
@ -44,23 +62,24 @@ impl FileInfo {
serde_json::from_str(&str_repr).map_err(FileError::from)
}
*/
pub fn from_path(path: &Path) -> Result<FileInfo, FileError> {
pub fn from_path(path: &Path) -> Result<FileInfo, ReadFileError> {
match (path.is_file(), path.is_dir()) {
(false, false) => Err(FileError::FileNotFound(PathBuf::from(path))),
(false, true) => Err(FileError::NotAFile(PathBuf::from(path))),
(false, false) => Err(ReadFileError::FileNotFound),
(false, true) => Err(ReadFileError::NotAFile),
(true, _) => Ok(()),
}?;
let metadata = path.metadata().map_err(FileError::IOError)?;
let metadata = path.metadata().map_err(ReadFileError::IOError)?;
let id = path
.file_name()
.map(|s| String::from(s.to_string_lossy()))
.ok_or(FileError::NotAFile(PathBuf::from(path)))?;
.ok_or(ReadFileError::NotAFile)?;
let created = metadata
.created()
.map(|m| DateTime::from(m))
.map_err(|err| FileError::IOError(err))?;
.map_err(|err| ReadFileError::IOError(err))?;
let file_type = String::from(
mime_guess::from_path(path)
.first_or_octet_stream()
@ -77,16 +96,17 @@ impl FileInfo {
})
}
fn hash_file(path: &Path) -> Result<HexString, FileError> {
fn hash_file(path: &Path) -> Result<HexString, ReadFileError> {
let mut buf = Vec::new();
let mut file = std::fs::File::open(path).map_err(FileError::from)?;
let mut file = std::fs::File::open(path).map_err(ReadFileError::from)?;
file.read_to_end(&mut buf).map_err(FileError::from)?;
file.read_to_end(&mut buf).map_err(ReadFileError::from)?;
let mut vec = Vec::new();
vec.extend_from_slice(Sha256::digest(&buf).as_slice());
Ok(HexString::from_bytes(&vec))
}
/*
fn metadata_path(id: &str, root: &Path) -> PathBuf {
let mut path = PathBuf::from(root);
path.push(".metadata");
@ -98,14 +118,23 @@ impl FileInfo {
let path = FileInfo::metadata_path(&self.id, &self.root);
remove_file(path).map_err(FileError::from)
}
*/
}
#[cfg(test)]
mod test {
use super::*;
use crate::lib::utils::FileCleanup;
struct FileContext(PathBuf);
impl FileRoot for FileContext {
fn root(&self) -> PathBuf {
self.0.clone()
}
}
/*
#[test]
fn it_generates_information_from_file_path() {
let path = Path::new("fixtures/rawr.png");
@ -131,17 +160,23 @@ mod test {
}
}
}
*/
#[test]
fn it_saves_and_loads_metadata() {
let path = Path::new("fixtures/rawr.png");
let _ = FileCleanup(append_extension(path, "json"));
let info = FileInfo::from_path(&path).unwrap();
info.save(Path::new("fixtures")).unwrap();
info.save(&PathResolver(PathBuf::from("fixtures/rawr.png")))
.unwrap();
assert!(Path::new("fixtures/.metadata/rawr.png.json").is_file());
let info_ = FileInfo::open("rawr.png", Path::new("fixtures")).unwrap();
let info_ = FileInfo::load(
FileId::from("rawr.png"),
FileContext(PathBuf::from("fixtures")),
)
.unwrap();
assert_eq!(info_.id, "rawr.png");
assert_eq!(info_.size, 23777);
assert_eq!(info_.created, info.created);

124
file-service/src/lib/mod.rs
View File

@ -1,4 +1,8 @@
use std::path::{Path, PathBuf};
use std::{
ops::Deref,
path::{Path, PathBuf},
};
use thiserror::Error;
use uuid::Uuid;
mod file;
@ -6,10 +10,98 @@ mod fileinfo;
mod thumbnail;
pub mod utils;
pub use file::{File, FileError};
pub use file::File;
pub use fileinfo::FileInfo;
pub use thumbnail::Thumbnail;
#[derive(Debug, Error)]
pub enum WriteFileError {
#[error("root file path does not exist")]
RootNotFound,
#[error("permission denied")]
PermissionDenied,
#[error("JSON error")]
JSONError(#[from] serde_json::error::Error),
#[error("IO error")]
IOError(#[from] std::io::Error),
}
#[derive(Debug, Error)]
pub enum ReadFileError {
#[error("file not found")]
FileNotFound,
#[error("path is not a file")]
NotAFile,
#[error("permission denied")]
PermissionDenied,
#[error("JSON error")]
JSONError(#[from] serde_json::error::Error),
#[error("IO error")]
IOError(#[from] std::io::Error),
}
#[derive(Clone, Debug)]
pub struct PathResolver(pub PathBuf);
impl PathResolver {
fn file_path(&self) -> PathBuf {
self.0.clone()
}
fn metadata_path(&self) -> PathBuf {
let mut path = self.0.clone();
path.set_extension("json");
path
}
fn thumbnail_path(&self) -> PathBuf {
let mut path = self.0.clone();
path.set_extension("tn");
path
}
}
#[derive(Clone, Debug)]
pub struct FileId(String);
impl From<String> for FileId {
fn from(s: String) -> Self {
Self(s)
}
}
impl From<&str> for FileId {
fn from(s: &str) -> Self {
Self(s.to_owned())
}
}
impl Deref for FileId {
type Target = String;
fn deref(&self) -> &Self::Target {
&self.0
}
}
pub trait FileRoot {
fn root(&self) -> PathBuf;
}
pub struct Context(PathBuf);
impl FileRoot for Context {
fn root(&self) -> PathBuf {
self.0.clone()
}
}
pub struct App {
files_root: PathBuf,
}
@ -21,19 +113,18 @@ impl App {
}
}
pub fn list_files(&self) -> Vec<Result<File, FileError>> {
File::list(&self.files_root)
pub fn list_files(&self) -> Vec<Result<File, ReadFileError>> {
unimplemented!()
}
pub fn add_file(
&mut self,
temp_path: &PathBuf,
filename: &Option<PathBuf>,
) -> Result<File, FileError> {
let id = Uuid::new_v4().hyphenated().to_string();
File::new(&id, &self.files_root, temp_path, filename)
pub fn add_file(&mut self, filename: String, content: Vec<u8>) -> Result<File, WriteFileError> {
let context = Context(self.files_root.clone());
let mut file = File::new(filename, context)?;
file.set_content(content)?;
Ok(file)
}
/*
pub fn delete_file(&mut self, id: String) -> Result<(), FileError> {
let f = File::open(&id, &self.files_root)?;
f.delete()
@ -42,17 +133,24 @@ impl App {
pub fn get_metadata(&self, id: String) -> Result<FileInfo, FileError> {
FileInfo::open(&id, &self.files_root)
}
*/
pub fn get_file(&self, id: &str) -> Result<(FileInfo, std::fs::File), FileError> {
pub fn get_file(&self, id: &str) -> Result<(FileInfo, std::fs::File), ReadFileError> {
/*
let f = File::open(&id, &self.files_root)?;
let info = f.info();
let stream = f.stream()?;
Ok((info, stream))
*/
unimplemented!()
}
pub fn get_thumbnail(&self, id: &str) -> Result<(FileInfo, std::fs::File), FileError> {
pub fn get_thumbnail(&self, id: &str) -> Result<(FileInfo, std::fs::File), ReadFileError> {
/*
let f = File::open(id, &self.files_root)?;
let stream = f.thumbnail().stream()?;
Ok((f.info(), stream))
*/
unimplemented!()
}
}

25
file-service/src/lib/thumbnail.rs
View File

@ -2,7 +2,7 @@ use image::imageops::FilterType;
use std::fs::remove_file;
use std::path::{Path, PathBuf};
use crate::FileError;
use super::{ReadFileError, WriteFileError};
#[derive(Clone, Debug, PartialEq)]
pub struct Thumbnail {
@ -11,7 +11,8 @@ pub struct Thumbnail {
}
impl Thumbnail {
pub fn open(id: &str, root: &Path) -> Result<Thumbnail, FileError> {
pub fn open(id: &str, root: &Path) -> Result<Thumbnail, WriteFileError> {
/*
let mut source_path = PathBuf::from(root);
source_path.push(id);
@ -28,20 +29,24 @@ impl Thumbnail {
}
Ok(self_)
*/
unimplemented!()
}
pub fn from_path(path: &Path) -> Result<Thumbnail, FileError> {
/*
pub fn from_path(path: &Path) -> Result<Thumbnail, ReadFileError> {
let id = path
.file_name()
.map(|s| String::from(s.to_string_lossy()))
.ok_or(FileError::NotAnImage(PathBuf::from(path)))?;
.ok_or(ReadFileError::NotAnImage(PathBuf::from(path)))?;
let root = path
.parent()
.ok_or(FileError::FileNotFound(PathBuf::from(path)))?;
.ok_or(ReadFileError::FileNotFound(PathBuf::from(path)))?;
Thumbnail::open(&id, root)
}
*/
fn thumbnail_path(id: &str, root: &Path) -> PathBuf {
let mut path = PathBuf::from(root);
@ -50,20 +55,20 @@ impl Thumbnail {
path
}
pub fn stream(&self) -> Result<std::fs::File, FileError> {
pub fn stream(&self) -> Result<std::fs::File, ReadFileError> {
let thumbnail_path = Thumbnail::thumbnail_path(&self.id, &self.root);
std::fs::File::open(thumbnail_path.clone()).map_err(|err| {
if err.kind() == std::io::ErrorKind::NotFound {
FileError::FileNotFound(thumbnail_path)
ReadFileError::FileNotFound
} else {
FileError::from(err)
ReadFileError::from(err)
}
})
}
pub fn delete(&self) -> Result<(), FileError> {
pub fn delete(&self) -> Result<(), WriteFileError> {
let path = Thumbnail::thumbnail_path(&self.id, &self.root);
remove_file(path).map_err(FileError::from)
remove_file(path).map_err(WriteFileError::from)
}
}

53
file-service/src/main.rs
View File

@ -13,6 +13,8 @@ use http::status::StatusCode;
// use mustache::{compile_path, Template};
// use orizentic::{Permissions, ResourceName, Secret};
use build_html::Html;
use bytes::Buf;
use futures_util::{Stream, StreamExt};
use std::{
collections::HashMap,
io::Read,
@ -20,7 +22,7 @@ use std::{
path::Path,
sync::{Arc, RwLock},
};
use warp::Filter;
use warp::{filters::multipart::Part, Filter};
mod cookies;
mod html;
@ -28,7 +30,7 @@ mod lib;
mod middleware;
mod pages;
use lib::{utils::append_extension, App, File, FileError, FileInfo};
use lib::{utils::append_extension, App, File, FileInfo};
/*
fn is_admin(resource: &ResourceName, permissions: &Permissions) -> bool {
@ -247,6 +249,32 @@ fn serve_file(
}
}
async fn collect_content(mut part: Part) -> Result<(Option<String>, Vec<u8>), String> {
let mut content: Vec<u8> = Vec::new();
while let Some(Ok(data)) = part.data().await {
let mut reader = data.reader();
reader.read_to_end(&mut content).unwrap();
}
Ok((part.filename().map(|s| s.to_owned()), content))
}
async fn collect_multipart(
mut stream: warp::filters::multipart::FormData,
) -> Result<Vec<(Option<String>, Vec<u8>)>, warp::Error> {
let mut content: Vec<(Option<String>, Vec<u8>)> = Vec::new();
while let Some(part) = stream.next().await {
match part {
Ok(part) => content.push(collect_content(part).await.unwrap()),
Err(err) => return Err(err),
}
}
Ok(content)
}
#[tokio::main]
pub async fn main() {
/*
@ -309,6 +337,7 @@ pub async fn main() {
}
});
/*
let post_handler = warp::path!(String)
.and(warp::post())
.and(warp::filters::body::form())
@ -320,6 +349,7 @@ pub async fn main() {
.status(StatusCode::SEE_OTHER)
.body(vec![])
});
*/
let thumbnail = warp::path!(String / "tn")
.and(warp::get())
@ -351,8 +381,20 @@ pub async fn main() {
}
});
let upload = warp::path!().and(warp::post()).map(|| {
println!("upload");
let upload = warp::path!()
.and(warp::post())
.and(warp::filters::multipart::form().max_length(1024 * 1024 * 32))
.then(|form: warp::filters::multipart::FormData| async move {
let files = collect_multipart(form).await;
/*
for (filename, content) in files {
app.write()
.unwrap()
.add_file(Some(filename), content)
.unwrap();
}
*/
println!("file length: {:?}", files.map(|f| f.len()));
warp::reply()
});
@ -361,6 +403,7 @@ pub async fn main() {
warp::reply()
});
/*
let server = warp::serve(
root.or(post_handler)
.or(file)
@ -369,6 +412,8 @@ pub async fn main() {
.or(delete)
.with(log),
);
*/
let server = warp::serve(root.or(upload).with(log));
server
.run(SocketAddr::new(IpAddr::V4(Ipv4Addr::new(0, 0, 0, 0)), 8002))
.await;

10
file-service/src/pages.rs
View File

@ -1,7 +1,7 @@
use crate::{html::*, File, FileError};
use crate::{html::*, lib::ReadFileError, File};
use build_html::{self, Container, ContainerType, Html, HtmlContainer};
pub fn index(files: Vec<Result<File, FileError>>) -> build_html::HtmlPage {
pub fn index(files: Vec<Result<File, ReadFileError>>) -> build_html::HtmlPage {
let mut page = build_html::HtmlPage::new()
.with_title("Admin list of files")
.with_header(1, "Admin list of files")
@ -21,7 +21,7 @@ pub fn index(files: Vec<Result<File, FileError>>) -> build_html::HtmlPage {
let container = match file {
Ok(ref file) => thumbnail(file).with_html(
Form::new()
.with_path(&format!("/{}", file.info().id))
.with_path(&format!("/{}", *file.id))
.with_method("post")
.with_html(Input::new("hidden", "_method").with_value("delete"))
.with_html(Button::new("Delete")),
@ -41,8 +41,8 @@ pub fn thumbnail(file: &File) -> Container {
let tn = Container::new(ContainerType::Div)
.with_attributes(vec![("class", "thumbnail")])
.with_link(
format!("/{}", file.info().id),
Image::new(&format!("{}/tn", file.info().id)).to_html_string(),
format!("/{}", *file.id),
Image::new(&format!("{}/tn", *file.id)).to_html_string(),
);
container.add_html(tn);
container