savanni
/
monorepo
1
0
Fork 0
Code Issues 76 Pull Requests Projects 6 Releases Wiki Activity

Add some tests to verify that a file can be added to the system 

Still gutting a lot of the old code, but this MR focuses more on ensuring that a file can be added and that the metadata gets saved.
This commit is contained in:
Savanni D'Gerinel 2023-09-22 23:43:45 -04:00
parent 9787ed3e67
commit 22e25256a5
13 changed files with 221 additions and 1739 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

72
Cargo.lock generated
View File

@ -152,18 +152,6 @@ version = "2.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b4682ae6287fcf752ecaabbfcc7b6f9b72aa33933dc23a554d853aea8eea8635"
[[package]]
name = "block-buffer"
version = "0.7.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c0940dc441f31689269e10ac70eb1002a3a1d3ad1390e030043662eb7fe4688b"
dependencies = [
"block-padding",
"byte-tools",
"byteorder",
"generic-array 0.12.4",
]
[[package]]
name = "block-buffer"
version = "0.10.4"
@ -173,15 +161,6 @@ dependencies = [
"generic-array 0.14.7",
]
[[package]]
name = "block-padding"
version = "0.1.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "fa79dedbb091f449f1f39e53edf88d5dbe95f895dae6135a8d7b881fb5af73f5"
dependencies = [
"byte-tools",
]
[[package]]
name = "bodyparser"
version = "0.8.0"
@ -217,12 +196,6 @@ version = "3.14.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7f30e7476521f6f8af1a1c4c0b8cc94f0bee37d91763d0ca2665f299b6cd8aec"
[[package]]
name = "byte-tools"
version = "0.3.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e3b5ca7a04898ad4bcd41c90c5285445ff5b791899bb1b0abdd2a2aa791211d7"
[[package]]
name = "bytemuck"
version = "1.14.0"
@ -539,22 +512,13 @@ dependencies = [
"byteorder",
]
[[package]]
name = "digest"
version = "0.8.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f3d0c8c8752312f9713efd397ff63acb9f85585afbf179282e720e7704954dd5"
dependencies = [
"generic-array 0.12.4",
]
[[package]]
name = "digest"
version = "0.10.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292"
dependencies = [
"block-buffer 0.10.4",
"block-buffer",
"crypto-common",
]
@ -667,12 +631,6 @@ dependencies = [
"zune-inflate",
]
[[package]]
name = "fake-simd"
version = "0.1.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e88a8acf291dafb59c2d96e8f59828f3838bb1a70398823ade51a84de6a6deed"
[[package]]
name = "fastrand"
version = "2.0.1"
@ -1002,15 +960,6 @@ dependencies = [
"typenum",
]
[[package]]
name = "generic-array"
version = "0.12.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ffdf9f34f1447443d37393cc6c2b8313aebddcd96906caf34e54c68d8e57d7bd"
dependencies = [
"typenum",
]
[[package]]
name = "generic-array"
version = "0.14.7"
@ -2307,12 +2256,6 @@ version = "1.18.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "dd8b5dd2ae5ed71462c540258bedcb51965123ad7e7ccf4b9a8cafaa4a63576d"
[[package]]
name = "opaque-debug"
version = "0.2.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2839e79665f131bdb5782e51f2c6c9599c133c6098982a54c794358bf432529c"
[[package]]
name = "openssl"
version = "0.10.57"
@ -3284,19 +3227,18 @@ checksum = "e3bf829a2d51ab4a5ddf1352d8470c140cadc8301b2ae1789db023f01cedd6ba"
dependencies = [
"cfg-if",
"cpufeatures",
"digest 0.10.7",
"digest",
]
[[package]]
name = "sha2"
version = "0.8.2"
version = "0.10.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a256f46ea78a0c0d9ff00077504903ac881a1dafdc20da66545699e7776b3e69"
checksum = "479fb9d862239e610720565ca91403019f2f00410f1864c5aa7479b950a76ed8"
dependencies = [
"block-buffer 0.7.3",
"digest 0.8.1",
"fake-simd",
"opaque-debug",
"cfg-if",
"cpufeatures",
"digest",
]
[[package]]

1
file-service/.ignore
View File

@ -1 +1,2 @@
fixtures
var

1404
file-service/Cargo.lock generated
View File

File diff suppressed because it is too large Load Diff

2
file-service/Cargo.toml
View File

@ -22,7 +22,7 @@ params = "*"
router = "*"
serde_json = "*"
serde = { version = "1.0", features = ["derive"] }
sha2 = "0.8.2"
sha2 = "0.10"
thiserror = "1.0.20"
tokio = { version = "1", features = [ "full" ] }
uuid = { version = "0.4", features = [ "serde", "v4" ] }

194
file-service/src/lib/fileinfo.rs
View File

@ -1,194 +0,0 @@
use chrono::prelude::*;
use hex_string::HexString;
use serde::{Deserialize, Serialize};
use serde_json;
use sha2::{Digest, Sha256};
use std::fs::remove_file;
use std::io::{Read, Write};
use std::path::{Path, PathBuf};
use super::{FileId, FileRoot, PathResolver, ReadFileError};
use crate::append_extension;
#[derive(Clone, Debug, Serialize, Deserialize)]
pub struct FileInfo {
pub id: String,
pub size: u64,
pub created: DateTime<Utc>,
pub file_type: String,
pub hash: String,
#[serde(skip)]
pub root: PathBuf,
}
impl FileInfo {
pub fn load<CTX: FileRoot>(id: FileId, context: CTX) -> Result<Self, ReadFileError> {
let mut content: Vec<u8> = Vec::new();
let mut file = std::fs::File::open(Self::path(id, context))?;
file.read_to_end(&mut content)?;
let js = serde_json::from_slice(&content)?;
Ok(js)
}
pub fn path<CTX: FileRoot>(id: FileId, context: CTX) -> PathBuf {
let mut path = context.root();
path.push((*id).to_owned());
path.set_extension("json");
path
}
pub fn save(&self, root: &PathResolver) -> Result<(), ReadFileError> {
let ser = serde_json::to_string(self).unwrap();
std::fs::File::create(root.metadata_path())
.and_then(|mut stream| stream.write(ser.as_bytes()).map(|_| (())))
.map_err(ReadFileError::from)
}
/*
pub fn open(id: &str, root: &Path) -> Result<FileInfo, FileError> {
let mut buf = Vec::new();
let md_path = FileInfo::metadata_path(id, root);
std::fs::File::open(md_path.clone())
.and_then(|mut stream| stream.read_to_end(&mut buf))
.map_err(move |err| match err.kind() {
std::io::ErrorKind::NotFound => FileError::FileNotFound(md_path),
_ => FileError::IOError(err),
})?;
let str_repr = std::str::from_utf8(&buf)?;
serde_json::from_str(&str_repr).map_err(FileError::from)
}
*/
pub fn from_path(path: &Path) -> Result<FileInfo, ReadFileError> {
match (path.is_file(), path.is_dir()) {
(false, false) => Err(ReadFileError::FileNotFound),
(false, true) => Err(ReadFileError::NotAFile),
(true, _) => Ok(()),
}?;
let metadata = path.metadata().map_err(ReadFileError::IOError)?;
let id = path
.file_name()
.map(|s| String::from(s.to_string_lossy()))
.ok_or(ReadFileError::NotAFile)?;
let created = metadata
.created()
.map(|m| DateTime::from(m))
.map_err(|err| ReadFileError::IOError(err))?;
let file_type = String::from(
mime_guess::from_path(path)
.first_or_octet_stream()
.essence_str(),
);
let hash = FileInfo::hash_file(path)?;
Ok(FileInfo {
id,
size: metadata.len(),
created,
file_type,
hash: hash.as_string(),
root: PathBuf::from(path.parent().unwrap()),
})
}
fn hash_file(path: &Path) -> Result<HexString, ReadFileError> {
let mut buf = Vec::new();
let mut file = std::fs::File::open(path).map_err(ReadFileError::from)?;
file.read_to_end(&mut buf).map_err(ReadFileError::from)?;
let mut vec = Vec::new();
vec.extend_from_slice(Sha256::digest(&buf).as_slice());
Ok(HexString::from_bytes(&vec))
}
/*
fn metadata_path(id: &str, root: &Path) -> PathBuf {
let mut path = PathBuf::from(root);
path.push(".metadata");
path.push(id.clone());
append_extension(&path, "json")
}
pub fn delete(&self) -> Result<(), FileError> {
let path = FileInfo::metadata_path(&self.id, &self.root);
remove_file(path).map_err(FileError::from)
}
*/
}
#[cfg(test)]
mod test {
use super::*;
use crate::lib::utils::FileCleanup;
struct FileContext(PathBuf);
impl FileRoot for FileContext {
fn root(&self) -> PathBuf {
self.0.clone()
}
}
/*
#[test]
fn it_generates_information_from_file_path() {
let path = Path::new("fixtures/rawr.png");
match FileInfo::from_path(&path) {
Ok(FileInfo {
id,
size,
file_type,
hash,
..
}) => {
assert_eq!(id, "rawr.png");
assert_eq!(size, 23777);
assert_eq!(file_type, "image/png");
assert_eq!(
hash,
"b6cd35e113b95d62f53d9cbd27ccefef47d3e324aef01a2db6c0c6d3a43c89ee"
);
}
Err(err) => {
println!("error loading file path: {}", err);
assert!(false);
}
}
}
*/
#[test]
fn it_saves_and_loads_metadata() {
let path = Path::new("fixtures/rawr.png");
let _ = FileCleanup(append_extension(path, "json"));
let info = FileInfo::from_path(&path).unwrap();
info.save(&PathResolver(PathBuf::from("fixtures/rawr.png")))
.unwrap();
assert!(Path::new("fixtures/.metadata/rawr.png.json").is_file());
let info_ = FileInfo::load(
FileId::from("rawr.png"),
FileContext(PathBuf::from("fixtures")),
)
.unwrap();
assert_eq!(info_.id, "rawr.png");
assert_eq!(info_.size, 23777);
assert_eq!(info_.created, info.created);
assert_eq!(info_.file_type, "image/png");
assert_eq!(info_.hash, info.hash);
}
#[test]
fn it_extends_a_file_extension() {
assert_eq!(
append_extension(Path::new("fixtures/rawr.png"), "json"),
Path::new("fixtures/rawr.png.json")
);
}
}

8
file-service/src/main.rs
View File

@ -19,18 +19,18 @@ use std::{
collections::HashMap,
io::Read,
net::{IpAddr, Ipv4Addr, SocketAddr},
path::Path,
path::PathBuf,
sync::{Arc, RwLock},
};
use warp::{filters::multipart::Part, Filter};
mod cookies;
mod html;
mod lib;
mod middleware;
mod pages;
mod store;
use lib::{utils::append_extension, App, File, FileInfo};
pub use store::{FileInfo, Store};
/*
fn is_admin(resource: &ResourceName, permissions: &Permissions) -> bool {
@ -320,7 +320,7 @@ pub async fn main() {
pretty_env_logger::init();
let app = Arc::new(RwLock::new(App::new(Path::new(
let app = Arc::new(RwLock::new(Store::new(PathBuf::from(
&std::env::var("FILE_SHARE_DIR").unwrap(),
))));

5
file-service/src/pages.rs
View File

@ -1,4 +1,7 @@
use crate::{html::*, lib::ReadFileError, File};
use crate::{
html::*,
store::{File, ReadFileError},
};
use build_html::{self, Container, ContainerType, Html, HtmlContainer};
pub fn index(files: Vec<Result<File, ReadFileError>>) -> build_html::HtmlPage {

95
file-service/src/lib/file.rs → file-service/src/store/file.rs
View File

@ -3,51 +3,21 @@ use super::{
WriteFileError,
};
use chrono::prelude::*;
use hex_string::HexString;
use sha2::{Digest, Sha256};
use std::{
fs::{copy, read_dir, remove_file},
io::{Read, Write},
path::{Path, PathBuf},
};
use thiserror::Error;
use uuid::Uuid;
/*
#[derive(Error, Debug)]
pub enum FileError {
#[error("not implemented")]
NotImplemented,
#[error("file not found: `{0}`")]
FileNotFound(PathBuf),
#[error("file is not an image: `{0}`")]
NotAnImage(PathBuf),
#[error("path is not a file: `{0}`")]
NotAFile(PathBuf),
#[error("Image loading error")]
ImageError(#[from] image::ImageError),
#[error("IO error")]
IOError(#[from] std::io::Error),
#[error("JSON error")]
JSONError(#[from] serde_json::error::Error),
#[error("UTF8 Error")]
UTF8Error(#[from] std::str::Utf8Error),
}
*/
/// One file in the database, complete with the path of the file and information about the
/// thumbnail of the file.
#[derive(Debug)]
pub struct File {
pub id: FileId,
path: PathResolver,
pub path: PathResolver,
pub info: FileInfo,
tn: Option<Thumbnail>,
}
impl File {
@ -65,41 +35,44 @@ impl File {
.to_owned();
let info = FileInfo {
id: (*id).to_owned(),
size: 0,
created: Utc::now(),
file_type,
hash: "".to_owned(),
root: context.root().to_owned(),
};
let mut md_file = std::fs::File::create(path.metadata_path())?;
md_file.write(&serde_json::to_vec(&info)?)?;
Ok(Self {
id,
path,
info,
tn: None,
})
Ok(Self { id, path, info })
}
pub fn load<CTX: FileRoot>(id: FileId, context: CTX) -> Result<Self, ReadFileError> {
let mut path = context.root();
path.push((*id).to_owned());
let path = PathResolver(path);
pub fn load(resolver: PathResolver) -> Result<Self, ReadFileError> {
/*
Ok(Self {
id: id.clone(),
path,
info: FileInfo::load(id, context)?,
tn: None,
id: FileId::from(
resolver
.file_path()
.file_stem()
.unwrap()
.to_string_lossy()
.to_owned(),
),
path: resolver,
info: FileInfo::load(resolver.metadata_path())?,
})
*/
unimplemented!()
}
pub fn set_content(&self, content: Vec<u8>) -> Result<(), WriteFileError> {
pub fn set_content(&mut self, content: Vec<u8>) -> Result<(), WriteFileError> {
let mut content_file = std::fs::File::create(self.path.file_path())?;
content_file.write(&content)?;
let byte_count = content_file.write(&content)?;
self.info.size = byte_count;
let mut md_file = std::fs::File::create(self.path.metadata_path())?;
md_file.write(&serde_json::to_vec(&self.info)?)?;
Ok(())
}
@ -107,6 +80,14 @@ impl File {
unimplemented!()
}
pub fn hash_content(&self) -> Result<HexString, ReadFileError> {
let mut buf = Vec::new();
let mut file = std::fs::File::open(self.path.file_path())?;
file.read_to_end(&mut buf).map_err(ReadFileError::from)?;
Ok(HexString::from_bytes(&Sha256::digest(&buf).to_vec()))
}
/*
pub fn new(
id: &str,
@ -208,8 +189,8 @@ impl File {
#[cfg(test)]
mod test {
use super::*;
use crate::lib::utils::FileCleanup;
use std::path::{Path, PathBuf};
use crate::store::utils::FileCleanup;
use std::path::PathBuf;
struct FileContext(PathBuf);
@ -261,10 +242,8 @@ mod test {
#[test]
fn it_raises_an_error_when_file_not_found() {
match File::load(
FileId::from("rawr.png"),
FileContext(PathBuf::from("fixtures/")),
) {
let resolver = PathResolver(PathBuf::from("fixtures/rawr.png"));
match File::load(resolver) {
Err(ReadFileError::FileNotFound) => assert!(true),
_ => assert!(false),
}

122
file-service/src/store/fileinfo.rs Normal file
View File

@ -0,0 +1,122 @@
use chrono::prelude::*;
use serde::{Deserialize, Serialize};
use serde_json;
use std::io::{Read, Write};
use std::path::PathBuf;
use super::{ReadFileError, WriteFileError};
#[derive(Clone, Debug, Serialize, Deserialize)]
pub struct FileInfo {
pub size: usize,
pub created: DateTime<Utc>,
pub file_type: String,
pub hash: String,
}
impl FileInfo {
pub fn load(path: PathBuf) -> Result<Self, ReadFileError> {
let mut content: Vec<u8> = Vec::new();
let mut file = std::fs::File::open(path)?;
file.read_to_end(&mut content)?;
let js = serde_json::from_slice(&content)?;
Ok(js)
}
pub fn save(&self, path: PathBuf) -> Result<(), WriteFileError> {
let ser = serde_json::to_string(self).unwrap();
let mut file = std::fs::File::create(path)?;
file.write(ser.as_bytes())?;
Ok(())
}
/*
pub fn from_path(path: &Path) -> Result<FileInfo, ReadFileError> {
match (path.is_file(), path.is_dir()) {
(false, false) => Err(ReadFileError::FileNotFound),
(false, true) => Err(ReadFileError::NotAFile),
(true, _) => Ok(()),
}?;
let metadata = path.metadata().map_err(ReadFileError::IOError)?;
let id = path
.file_name()
.map(|s| String::from(s.to_string_lossy()))
.ok_or(ReadFileError::NotAFile)?;
let created = metadata
.created()
.map(|m| DateTime::from(m))
.map_err(|err| ReadFileError::IOError(err))?;
let file_type = String::from(
mime_guess::from_path(path)
.first_or_octet_stream()
.essence_str(),
);
let hash = FileInfo::hash_file(path)?;
Ok(FileInfo {
id,
size: metadata.len(),
created,
file_type,
hash: hash.as_string(),
root: PathBuf::from(path.parent().unwrap()),
})
}
*/
/*
fn hash_file(path: &Path) -> Result<HexString, ReadFileError> {
let mut buf = Vec::new();
let mut file = std::fs::File::open(path).map_err(ReadFileError::from)?;
file.read_to_end(&mut buf).map_err(ReadFileError::from)?;
let mut vec = Vec::new();
vec.extend_from_slice(Sha256::digest(&buf).as_slice());
Ok(HexString::from_bytes(&vec))
}
*/
/*
fn metadata_path(id: &str, root: &Path) -> PathBuf {
let mut path = PathBuf::from(root);
path.push(".metadata");
path.push(id.clone());
append_extension(&path, "json")
}
pub fn delete(&self) -> Result<(), FileError> {
let path = FileInfo::metadata_path(&self.id, &self.root);
remove_file(path).map_err(FileError::from)
}
*/
}
#[cfg(test)]
mod test {
use super::*;
use crate::store::utils::FileCleanup;
use crate::store::PathResolver;
#[test]
fn it_saves_and_loads_metadata() {
let resolver = PathResolver::from("fixtures/1617654d-a588-4714-b4fa-e00ed0a8a607");
let _cleanup = FileCleanup(resolver.metadata_path());
let created = Utc::now();
let info = FileInfo {
size: 23777,
created,
file_type: "image/png".to_owned(),
hash: "abcdefg".to_owned(),
};
info.save(resolver.metadata_path()).unwrap();
let info_ = FileInfo::load(resolver.metadata_path()).unwrap();
assert_eq!(info_.size, 23777);
assert_eq!(info_.created, info.created);
assert_eq!(info_.file_type, "image/png");
assert_eq!(info_.hash, info.hash);
}
}

53
file-service/src/lib/mod.rs → file-service/src/store/mod.rs
View File

@ -114,15 +114,13 @@ impl FileRoot for Context {
}
}
pub struct App {
pub struct Store {
files_root: PathBuf,
}
impl App {
pub fn new(files_root: &Path) -> App {
App {
files_root: PathBuf::from(files_root),
}
impl Store {
pub fn new(files_root: PathBuf) -> Self {
Self { files_root }
}
pub fn list_files(&self) -> Vec<Result<File, ReadFileError>> {
@ -136,16 +134,18 @@ impl App {
Ok(file)
}
pub fn delete_file(&mut self, id: String) -> Result<(), WriteFileError> {
/*
pub fn delete_file(&mut self, id: String) -> Result<(), FileError> {
let f = File::open(&id, &self.files_root)?;
f.delete()
*/
unimplemented!()
}
pub fn get_metadata(&self, id: String) -> Result<FileInfo, FileError> {
FileInfo::open(&id, &self.files_root)
pub fn get_metadata(&self, id: String) -> Result<FileInfo, ReadFileError> {
// FileInfo::open(&id, &self.files_root)
unimplemented!()
}
*/
pub fn get_file(&self, id: &str) -> Result<(FileInfo, std::fs::File), ReadFileError> {
/*
@ -169,7 +169,10 @@ impl App {
#[cfg(test)]
mod test {
use crate::store::utils::FileCleanup;
use super::*;
use std::io::Read;
#[test]
fn paths() {
@ -187,4 +190,34 @@ mod test {
PathBuf::from("path/82420255-d3c8-4d90-a582-f94be588c70c.tn")
);
}
#[test]
fn adds_files() {
let mut buf = Vec::new();
let mut file = std::fs::File::open("fixtures/rawr.png").unwrap();
file.read_to_end(&mut buf).unwrap();
let mut store = Store::new(PathBuf::from("var/"));
let file_record = store.add_file("rawr.png".to_owned(), buf).unwrap();
let _file = FileCleanup(PathBuf::from(format!("var/{}", *file_record.id)));
let _md = FileCleanup(PathBuf::from(format!("var/{}.json", *file_record.id)));
let _tn = FileCleanup(PathBuf::from(format!("var/{}.tn", *file_record.id)));
assert!(PathBuf::from(format!("var/{}", *file_record.id)).exists());
assert!(PathBuf::from(format!("var/{}.json", *file_record.id)).exists());
assert!(PathBuf::from(format!("var/{}.tn", *file_record.id)).exists());
}
#[test]
fn sets_up_metadata_for_file() {}
#[test]
fn sets_up_thumbnail_for_file() {}
#[test]
fn deletes_associated_files() {}
#[test]
fn lists_files_in_the_db() {}
}

2
file-service/src/lib/thumbnail.rs → file-service/src/store/thumbnail.rs
View File

@ -75,7 +75,7 @@ impl Thumbnail {
#[cfg(test)]
mod test {
use super::*;
use crate::lib::utils::FileCleanup;
use crate::store::utils::FileCleanup;
#[test]
fn it_creates_a_thumbnail_if_one_does_not_exist() {

0
file-service/src/lib/utils.rs → file-service/src/store/utils.rs
View File

0
file-service/var/.placeholder Normal file
View File