Add some tests to verify that a file can be added to the system

Still gutting a lot of the old code, but this MR focuses more on ensuring that a file can be added and that the metadata gets saved.
This commit is contained in:
Savanni D'Gerinel 2023-09-22 23:43:45 -04:00
parent 163e1e1de1
commit 90f9b80dd9
13 changed files with 221 additions and 1739 deletions

72
Cargo.lock generated
View File

@ -158,18 +158,6 @@ version = "2.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b4682ae6287fcf752ecaabbfcc7b6f9b72aa33933dc23a554d853aea8eea8635"
[[package]]
name = "block-buffer"
version = "0.7.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c0940dc441f31689269e10ac70eb1002a3a1d3ad1390e030043662eb7fe4688b"
dependencies = [
"block-padding",
"byte-tools",
"byteorder",
"generic-array 0.12.4",
]
[[package]]
name = "block-buffer"
version = "0.10.4"
@ -179,15 +167,6 @@ dependencies = [
"generic-array 0.14.7",
]
[[package]]
name = "block-padding"
version = "0.1.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "fa79dedbb091f449f1f39e53edf88d5dbe95f895dae6135a8d7b881fb5af73f5"
dependencies = [
"byte-tools",
]
[[package]]
name = "bodyparser"
version = "0.8.0"
@ -223,12 +202,6 @@ version = "3.13.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a3e2c3daef883ecc1b5d58c15adae93470a91d425f3532ba1695849656af3fc1"
[[package]]
name = "byte-tools"
version = "0.3.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e3b5ca7a04898ad4bcd41c90c5285445ff5b791899bb1b0abdd2a2aa791211d7"
[[package]]
name = "bytemuck"
version = "1.13.1"
@ -550,22 +523,13 @@ dependencies = [
"byteorder",
]
[[package]]
name = "digest"
version = "0.8.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f3d0c8c8752312f9713efd397ff63acb9f85585afbf179282e720e7704954dd5"
dependencies = [
"generic-array 0.12.4",
]
[[package]]
name = "digest"
version = "0.10.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292"
dependencies = [
"block-buffer 0.10.4",
"block-buffer",
"crypto-common",
]
@ -678,12 +642,6 @@ dependencies = [
"zune-inflate",
]
[[package]]
name = "fake-simd"
version = "0.1.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e88a8acf291dafb59c2d96e8f59828f3838bb1a70398823ade51a84de6a6deed"
[[package]]
name = "fastrand"
version = "2.0.0"
@ -1017,15 +975,6 @@ dependencies = [
"typenum",
]
[[package]]
name = "generic-array"
version = "0.12.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ffdf9f34f1447443d37393cc6c2b8313aebddcd96906caf34e54c68d8e57d7bd"
dependencies = [
"typenum",
]
[[package]]
name = "generic-array"
version = "0.14.7"
@ -2325,12 +2274,6 @@ version = "1.18.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "dd8b5dd2ae5ed71462c540258bedcb51965123ad7e7ccf4b9a8cafaa4a63576d"
[[package]]
name = "opaque-debug"
version = "0.2.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2839e79665f131bdb5782e51f2c6c9599c133c6098982a54c794358bf432529c"
[[package]]
name = "openssl"
version = "0.10.57"
@ -3309,19 +3252,18 @@ checksum = "f04293dc80c3993519f2d7f6f511707ee7094fe0c6d3406feb330cdb3540eba3"
dependencies = [
"cfg-if",
"cpufeatures",
"digest 0.10.7",
"digest",
]
[[package]]
name = "sha2"
version = "0.8.2"
version = "0.10.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a256f46ea78a0c0d9ff00077504903ac881a1dafdc20da66545699e7776b3e69"
checksum = "479fb9d862239e610720565ca91403019f2f00410f1864c5aa7479b950a76ed8"
dependencies = [
"block-buffer 0.7.3",
"digest 0.8.1",
"fake-simd",
"opaque-debug",
"cfg-if",
"cpufeatures",
"digest",
]
[[package]]

View File

@ -1 +1,2 @@
fixtures
var

1404
file-service/Cargo.lock generated

File diff suppressed because it is too large Load Diff

View File

@ -22,7 +22,7 @@ params = "*"
router = "*"
serde_json = "*"
serde = { version = "1.0", features = ["derive"] }
sha2 = "0.8.2"
sha2 = "0.10"
thiserror = "1.0.20"
tokio = { version = "1", features = [ "full" ] }
uuid = { version = "0.4", features = [ "serde", "v4" ] }

View File

@ -1,194 +0,0 @@
use chrono::prelude::*;
use hex_string::HexString;
use serde::{Deserialize, Serialize};
use serde_json;
use sha2::{Digest, Sha256};
use std::fs::remove_file;
use std::io::{Read, Write};
use std::path::{Path, PathBuf};
use super::{FileId, FileRoot, PathResolver, ReadFileError};
use crate::append_extension;
#[derive(Clone, Debug, Serialize, Deserialize)]
pub struct FileInfo {
pub id: String,
pub size: u64,
pub created: DateTime<Utc>,
pub file_type: String,
pub hash: String,
#[serde(skip)]
pub root: PathBuf,
}
impl FileInfo {
pub fn load<CTX: FileRoot>(id: FileId, context: CTX) -> Result<Self, ReadFileError> {
let mut content: Vec<u8> = Vec::new();
let mut file = std::fs::File::open(Self::path(id, context))?;
file.read_to_end(&mut content)?;
let js = serde_json::from_slice(&content)?;
Ok(js)
}
pub fn path<CTX: FileRoot>(id: FileId, context: CTX) -> PathBuf {
let mut path = context.root();
path.push((*id).to_owned());
path.set_extension("json");
path
}
pub fn save(&self, root: &PathResolver) -> Result<(), ReadFileError> {
let ser = serde_json::to_string(self).unwrap();
std::fs::File::create(root.metadata_path())
.and_then(|mut stream| stream.write(ser.as_bytes()).map(|_| (())))
.map_err(ReadFileError::from)
}
/*
pub fn open(id: &str, root: &Path) -> Result<FileInfo, FileError> {
let mut buf = Vec::new();
let md_path = FileInfo::metadata_path(id, root);
std::fs::File::open(md_path.clone())
.and_then(|mut stream| stream.read_to_end(&mut buf))
.map_err(move |err| match err.kind() {
std::io::ErrorKind::NotFound => FileError::FileNotFound(md_path),
_ => FileError::IOError(err),
})?;
let str_repr = std::str::from_utf8(&buf)?;
serde_json::from_str(&str_repr).map_err(FileError::from)
}
*/
pub fn from_path(path: &Path) -> Result<FileInfo, ReadFileError> {
match (path.is_file(), path.is_dir()) {
(false, false) => Err(ReadFileError::FileNotFound),
(false, true) => Err(ReadFileError::NotAFile),
(true, _) => Ok(()),
}?;
let metadata = path.metadata().map_err(ReadFileError::IOError)?;
let id = path
.file_name()
.map(|s| String::from(s.to_string_lossy()))
.ok_or(ReadFileError::NotAFile)?;
let created = metadata
.created()
.map(|m| DateTime::from(m))
.map_err(|err| ReadFileError::IOError(err))?;
let file_type = String::from(
mime_guess::from_path(path)
.first_or_octet_stream()
.essence_str(),
);
let hash = FileInfo::hash_file(path)?;
Ok(FileInfo {
id,
size: metadata.len(),
created,
file_type,
hash: hash.as_string(),
root: PathBuf::from(path.parent().unwrap()),
})
}
fn hash_file(path: &Path) -> Result<HexString, ReadFileError> {
let mut buf = Vec::new();
let mut file = std::fs::File::open(path).map_err(ReadFileError::from)?;
file.read_to_end(&mut buf).map_err(ReadFileError::from)?;
let mut vec = Vec::new();
vec.extend_from_slice(Sha256::digest(&buf).as_slice());
Ok(HexString::from_bytes(&vec))
}
/*
fn metadata_path(id: &str, root: &Path) -> PathBuf {
let mut path = PathBuf::from(root);
path.push(".metadata");
path.push(id.clone());
append_extension(&path, "json")
}
pub fn delete(&self) -> Result<(), FileError> {
let path = FileInfo::metadata_path(&self.id, &self.root);
remove_file(path).map_err(FileError::from)
}
*/
}
#[cfg(test)]
mod test {
use super::*;
use crate::lib::utils::FileCleanup;
struct FileContext(PathBuf);
impl FileRoot for FileContext {
fn root(&self) -> PathBuf {
self.0.clone()
}
}
/*
#[test]
fn it_generates_information_from_file_path() {
let path = Path::new("fixtures/rawr.png");
match FileInfo::from_path(&path) {
Ok(FileInfo {
id,
size,
file_type,
hash,
..
}) => {
assert_eq!(id, "rawr.png");
assert_eq!(size, 23777);
assert_eq!(file_type, "image/png");
assert_eq!(
hash,
"b6cd35e113b95d62f53d9cbd27ccefef47d3e324aef01a2db6c0c6d3a43c89ee"
);
}
Err(err) => {
println!("error loading file path: {}", err);
assert!(false);
}
}
}
*/
#[test]
fn it_saves_and_loads_metadata() {
let path = Path::new("fixtures/rawr.png");
let _ = FileCleanup(append_extension(path, "json"));
let info = FileInfo::from_path(&path).unwrap();
info.save(&PathResolver(PathBuf::from("fixtures/rawr.png")))
.unwrap();
assert!(Path::new("fixtures/.metadata/rawr.png.json").is_file());
let info_ = FileInfo::load(
FileId::from("rawr.png"),
FileContext(PathBuf::from("fixtures")),
)
.unwrap();
assert_eq!(info_.id, "rawr.png");
assert_eq!(info_.size, 23777);
assert_eq!(info_.created, info.created);
assert_eq!(info_.file_type, "image/png");
assert_eq!(info_.hash, info.hash);
}
#[test]
fn it_extends_a_file_extension() {
assert_eq!(
append_extension(Path::new("fixtures/rawr.png"), "json"),
Path::new("fixtures/rawr.png.json")
);
}
}

View File

@ -19,18 +19,18 @@ use std::{
collections::HashMap,
io::Read,
net::{IpAddr, Ipv4Addr, SocketAddr},
path::Path,
path::PathBuf,
sync::{Arc, RwLock},
};
use warp::{filters::multipart::Part, Filter};
mod cookies;
mod html;
mod lib;
mod middleware;
mod pages;
mod store;
use lib::{utils::append_extension, App, File, FileInfo};
pub use store::{FileInfo, Store};
/*
fn is_admin(resource: &ResourceName, permissions: &Permissions) -> bool {
@ -320,7 +320,7 @@ pub async fn main() {
pretty_env_logger::init();
let app = Arc::new(RwLock::new(App::new(Path::new(
let app = Arc::new(RwLock::new(Store::new(PathBuf::from(
&std::env::var("FILE_SHARE_DIR").unwrap(),
))));

View File

@ -1,4 +1,7 @@
use crate::{html::*, lib::ReadFileError, File};
use crate::{
html::*,
store::{File, ReadFileError},
};
use build_html::{self, Container, ContainerType, Html, HtmlContainer};
pub fn index(files: Vec<Result<File, ReadFileError>>) -> build_html::HtmlPage {

View File

@ -3,51 +3,21 @@ use super::{
WriteFileError,
};
use chrono::prelude::*;
use hex_string::HexString;
use sha2::{Digest, Sha256};
use std::{
fs::{copy, read_dir, remove_file},
io::{Read, Write},
path::{Path, PathBuf},
};
use thiserror::Error;
use uuid::Uuid;
/*
#[derive(Error, Debug)]
pub enum FileError {
#[error("not implemented")]
NotImplemented,
#[error("file not found: `{0}`")]
FileNotFound(PathBuf),
#[error("file is not an image: `{0}`")]
NotAnImage(PathBuf),
#[error("path is not a file: `{0}`")]
NotAFile(PathBuf),
#[error("Image loading error")]
ImageError(#[from] image::ImageError),
#[error("IO error")]
IOError(#[from] std::io::Error),
#[error("JSON error")]
JSONError(#[from] serde_json::error::Error),
#[error("UTF8 Error")]
UTF8Error(#[from] std::str::Utf8Error),
}
*/
/// One file in the database, complete with the path of the file and information about the
/// thumbnail of the file.
#[derive(Debug)]
pub struct File {
pub id: FileId,
path: PathResolver,
pub path: PathResolver,
pub info: FileInfo,
tn: Option<Thumbnail>,
}
impl File {
@ -65,41 +35,44 @@ impl File {
.to_owned();
let info = FileInfo {
id: (*id).to_owned(),
size: 0,
created: Utc::now(),
file_type,
hash: "".to_owned(),
root: context.root().to_owned(),
};
let mut md_file = std::fs::File::create(path.metadata_path())?;
md_file.write(&serde_json::to_vec(&info)?)?;
Ok(Self {
id,
path,
info,
tn: None,
})
Ok(Self { id, path, info })
}
pub fn load<CTX: FileRoot>(id: FileId, context: CTX) -> Result<Self, ReadFileError> {
let mut path = context.root();
path.push((*id).to_owned());
let path = PathResolver(path);
pub fn load(resolver: PathResolver) -> Result<Self, ReadFileError> {
/*
Ok(Self {
id: id.clone(),
path,
info: FileInfo::load(id, context)?,
tn: None,
id: FileId::from(
resolver
.file_path()
.file_stem()
.unwrap()
.to_string_lossy()
.to_owned(),
),
path: resolver,
info: FileInfo::load(resolver.metadata_path())?,
})
*/
unimplemented!()
}
pub fn set_content(&self, content: Vec<u8>) -> Result<(), WriteFileError> {
pub fn set_content(&mut self, content: Vec<u8>) -> Result<(), WriteFileError> {
let mut content_file = std::fs::File::create(self.path.file_path())?;
content_file.write(&content)?;
let byte_count = content_file.write(&content)?;
self.info.size = byte_count;
let mut md_file = std::fs::File::create(self.path.metadata_path())?;
md_file.write(&serde_json::to_vec(&self.info)?)?;
Ok(())
}
@ -107,6 +80,14 @@ impl File {
unimplemented!()
}
pub fn hash_content(&self) -> Result<HexString, ReadFileError> {
let mut buf = Vec::new();
let mut file = std::fs::File::open(self.path.file_path())?;
file.read_to_end(&mut buf).map_err(ReadFileError::from)?;
Ok(HexString::from_bytes(&Sha256::digest(&buf).to_vec()))
}
/*
pub fn new(
id: &str,
@ -208,8 +189,8 @@ impl File {
#[cfg(test)]
mod test {
use super::*;
use crate::lib::utils::FileCleanup;
use std::path::{Path, PathBuf};
use crate::store::utils::FileCleanup;
use std::path::PathBuf;
struct FileContext(PathBuf);
@ -261,10 +242,8 @@ mod test {
#[test]
fn it_raises_an_error_when_file_not_found() {
match File::load(
FileId::from("rawr.png"),
FileContext(PathBuf::from("fixtures/")),
) {
let resolver = PathResolver(PathBuf::from("fixtures/rawr.png"));
match File::load(resolver) {
Err(ReadFileError::FileNotFound) => assert!(true),
_ => assert!(false),
}

View File

@ -0,0 +1,122 @@
use chrono::prelude::*;
use serde::{Deserialize, Serialize};
use serde_json;
use std::io::{Read, Write};
use std::path::PathBuf;
use super::{ReadFileError, WriteFileError};
#[derive(Clone, Debug, Serialize, Deserialize)]
pub struct FileInfo {
pub size: usize,
pub created: DateTime<Utc>,
pub file_type: String,
pub hash: String,
}
impl FileInfo {
pub fn load(path: PathBuf) -> Result<Self, ReadFileError> {
let mut content: Vec<u8> = Vec::new();
let mut file = std::fs::File::open(path)?;
file.read_to_end(&mut content)?;
let js = serde_json::from_slice(&content)?;
Ok(js)
}
pub fn save(&self, path: PathBuf) -> Result<(), WriteFileError> {
let ser = serde_json::to_string(self).unwrap();
let mut file = std::fs::File::create(path)?;
file.write(ser.as_bytes())?;
Ok(())
}
/*
pub fn from_path(path: &Path) -> Result<FileInfo, ReadFileError> {
match (path.is_file(), path.is_dir()) {
(false, false) => Err(ReadFileError::FileNotFound),
(false, true) => Err(ReadFileError::NotAFile),
(true, _) => Ok(()),
}?;
let metadata = path.metadata().map_err(ReadFileError::IOError)?;
let id = path
.file_name()
.map(|s| String::from(s.to_string_lossy()))
.ok_or(ReadFileError::NotAFile)?;
let created = metadata
.created()
.map(|m| DateTime::from(m))
.map_err(|err| ReadFileError::IOError(err))?;
let file_type = String::from(
mime_guess::from_path(path)
.first_or_octet_stream()
.essence_str(),
);
let hash = FileInfo::hash_file(path)?;
Ok(FileInfo {
id,
size: metadata.len(),
created,
file_type,
hash: hash.as_string(),
root: PathBuf::from(path.parent().unwrap()),
})
}
*/
/*
fn hash_file(path: &Path) -> Result<HexString, ReadFileError> {
let mut buf = Vec::new();
let mut file = std::fs::File::open(path).map_err(ReadFileError::from)?;
file.read_to_end(&mut buf).map_err(ReadFileError::from)?;
let mut vec = Vec::new();
vec.extend_from_slice(Sha256::digest(&buf).as_slice());
Ok(HexString::from_bytes(&vec))
}
*/
/*
fn metadata_path(id: &str, root: &Path) -> PathBuf {
let mut path = PathBuf::from(root);
path.push(".metadata");
path.push(id.clone());
append_extension(&path, "json")
}
pub fn delete(&self) -> Result<(), FileError> {
let path = FileInfo::metadata_path(&self.id, &self.root);
remove_file(path).map_err(FileError::from)
}
*/
}
#[cfg(test)]
mod test {
use super::*;
use crate::store::utils::FileCleanup;
use crate::store::PathResolver;
#[test]
fn it_saves_and_loads_metadata() {
let resolver = PathResolver::from("fixtures/1617654d-a588-4714-b4fa-e00ed0a8a607");
let _cleanup = FileCleanup(resolver.metadata_path());
let created = Utc::now();
let info = FileInfo {
size: 23777,
created,
file_type: "image/png".to_owned(),
hash: "abcdefg".to_owned(),
};
info.save(resolver.metadata_path()).unwrap();
let info_ = FileInfo::load(resolver.metadata_path()).unwrap();
assert_eq!(info_.size, 23777);
assert_eq!(info_.created, info.created);
assert_eq!(info_.file_type, "image/png");
assert_eq!(info_.hash, info.hash);
}
}

View File

@ -114,15 +114,13 @@ impl FileRoot for Context {
}
}
pub struct App {
pub struct Store {
files_root: PathBuf,
}
impl App {
pub fn new(files_root: &Path) -> App {
App {
files_root: PathBuf::from(files_root),
}
impl Store {
pub fn new(files_root: PathBuf) -> Self {
Self { files_root }
}
pub fn list_files(&self) -> Vec<Result<File, ReadFileError>> {
@ -136,16 +134,18 @@ impl App {
Ok(file)
}
pub fn delete_file(&mut self, id: String) -> Result<(), WriteFileError> {
/*
pub fn delete_file(&mut self, id: String) -> Result<(), FileError> {
let f = File::open(&id, &self.files_root)?;
f.delete()
*/
unimplemented!()
}
pub fn get_metadata(&self, id: String) -> Result<FileInfo, FileError> {
FileInfo::open(&id, &self.files_root)
pub fn get_metadata(&self, id: String) -> Result<FileInfo, ReadFileError> {
// FileInfo::open(&id, &self.files_root)
unimplemented!()
}
*/
pub fn get_file(&self, id: &str) -> Result<(FileInfo, std::fs::File), ReadFileError> {
/*
@ -169,7 +169,10 @@ impl App {
#[cfg(test)]
mod test {
use crate::store::utils::FileCleanup;
use super::*;
use std::io::Read;
#[test]
fn paths() {
@ -187,4 +190,34 @@ mod test {
PathBuf::from("path/82420255-d3c8-4d90-a582-f94be588c70c.tn")
);
}
#[test]
fn adds_files() {
let mut buf = Vec::new();
let mut file = std::fs::File::open("fixtures/rawr.png").unwrap();
file.read_to_end(&mut buf).unwrap();
let mut store = Store::new(PathBuf::from("var/"));
let file_record = store.add_file("rawr.png".to_owned(), buf).unwrap();
let _file = FileCleanup(PathBuf::from(format!("var/{}", *file_record.id)));
let _md = FileCleanup(PathBuf::from(format!("var/{}.json", *file_record.id)));
let _tn = FileCleanup(PathBuf::from(format!("var/{}.tn", *file_record.id)));
assert!(PathBuf::from(format!("var/{}", *file_record.id)).exists());
assert!(PathBuf::from(format!("var/{}.json", *file_record.id)).exists());
assert!(PathBuf::from(format!("var/{}.tn", *file_record.id)).exists());
}
#[test]
fn sets_up_metadata_for_file() {}
#[test]
fn sets_up_thumbnail_for_file() {}
#[test]
fn deletes_associated_files() {}
#[test]
fn lists_files_in_the_db() {}
}

View File

@ -75,7 +75,7 @@ impl Thumbnail {
#[cfg(test)]
mod test {
use super::*;
use crate::lib::utils::FileCleanup;
use crate::store::utils::FileCleanup;
#[test]
fn it_creates_a_thumbnail_if_one_does_not_exist() {

View File