neri
/
datatrash
1
0
Fork 1
Code Issues 7 Pull Requests Releases Wiki Activity

implement delete on download, better logging and multipart parsing 

a new database column was introduced for delete on download.
the model should be migrated automatically.

Fixes #4
This commit is contained in:
neri 2021-04-04 03:38:29 +02:00
parent c551f40c7a
commit 77fd25b787
6 changed files with 123 additions and 50 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
init-db.sql
View File

@ -6,3 +6,5 @@ CREATE TABLE IF NOT EXISTS files (
kind varchar(255) not null,
primary key (id)
);
ALTER TABLE files ADD COLUMN IF NOT EXISTS delete_on_download boolean;

40
src/deleter.rs
View File

@ -1,6 +1,11 @@
use async_std::{channel::Receiver, fs, path::PathBuf, task};
use async_std::{
channel::Receiver,
fs,
path::{Path, PathBuf},
task,
};
use chrono::{prelude::*, Duration};
use futures::{TryStreamExt, future::FutureExt};
use futures::{future::FutureExt, TryStreamExt};
use sqlx::{postgres::PgPool, Row};
pub(crate) async fn delete_old_files(receiver: Receiver<()>, db: PgPool, files_dir: PathBuf) {
@ -13,12 +18,9 @@ pub(crate) async fn delete_old_files(receiver: Receiver<()>, db: PgPool, files_d
.fetch(&db);
while let Some(row) = rows.try_next().await.expect("could not load expired files") {
let file_id: String = row.try_get("file_id").expect("we selected this column");
let mut path = files_dir.clone();
path.push(&file_id);
if path.exists().await {
log::info!("delete file {}", file_id);
fs::remove_file(&path).await.expect("could not delete file");
}
delete_content(&file_id, &files_dir)
.await
.expect("could not delete file");
}
sqlx::query("DELETE FROM files WHERE valid_till < $1")
@ -29,6 +31,28 @@ pub(crate) async fn delete_old_files(receiver: Receiver<()>, db: PgPool, files_d
}
}
pub(crate) async fn delete_by_id(
db: &PgPool,
file_id: &str,
files_dir: &Path,
) -> Result<(), sqlx::Error> {
delete_content(file_id, &files_dir).await?;
sqlx::query("DELETE FROM files WHERE file_id = $1")
.bind(file_id)
.execute(db)
.await?;
Ok(())
}
async fn delete_content(file_id: &str, files_dir: &Path) -> Result<(), std::io::Error> {
let path = files_dir.join(file_id);
if path.exists().await {
log::info!("delete file {}", file_id);
fs::remove_file(&path).await?;
}
Ok(())
}
async fn wait_for_file_expiry(receiver: &Receiver<()>, db: &PgPool) {
let mut rows = sqlx::query("SELECT MIN(valid_till) as min from files").fetch(db);
let row = rows

52
src/main.rs
View File

@ -15,9 +15,11 @@ use async_std::{
path::{Path, PathBuf},
task,
};
use env_logger::Env;
use file_kind::FileKind;
use futures::TryStreamExt;
use mime::Mime;
use multipart::UploadConfig;
use rand::prelude::SliceRandom;
use sqlx::{
postgres::{PgPool, PgPoolOptions, PgRow},
@ -58,7 +60,12 @@ async fn upload(
let parsed_multipart =
multipart::parse_multipart(payload, &file_id, &filename, config.max_file_size).await;
let (original_name, valid_till, kind) = match parsed_multipart {
let UploadConfig {
original_name,
valid_till,
kind,
delete_on_download,
} = match parsed_multipart {
Ok(data) => data,
Err(err) => {
if filename.exists().await {
@ -73,12 +80,14 @@ async fn upload(
};
let db_insert = sqlx::query(
"INSERT INTO Files (file_id, file_name, valid_till, kind) VALUES ($1, $2, $3, $4)",
"INSERT INTO Files (file_id, file_name, valid_till, kind, delete_on_download) \
VALUES ($1, $2, $3, $4, $5)",
)
.bind(&file_id)
.bind(original_name.as_ref().unwrap_or(&file_id))
.bind(&original_name)
.bind(valid_till.naive_local())
.bind(kind.to_string())
.bind(delete_on_download)
.execute(db.as_ref())
.await;
if db_insert.is_err() {
@ -93,22 +102,24 @@ async fn upload(
}
log::info!(
"create new file {} (valid_till: {}, kind: {})",
"{} create new file {} (valid_till: {}, kind: {}, delete_on_download: {})",
req.connection_info().realip_remote_addr().unwrap_or("-"),
file_id,
valid_till,
kind
kind,
delete_on_download
);
expiry_watch_sender.send(()).await.unwrap();
let redirect = if kind == FileKind::BINARY && original_name.is_some() {
let encoded_name = urlencoding::encode(original_name.as_ref().unwrap());
let redirect = if kind == FileKind::BINARY {
let encoded_name = urlencoding::encode(&original_name);
format!("/upload/{}/{}", file_id, encoded_name)
} else {
format!("/upload/{}", file_id)
};
let url = get_file_url(&req, &file_id, original_name.as_deref());
let url = get_file_url(&req, &file_id, Some(&original_name));
Ok(HttpResponse::SeeOther()
.header("location", redirect)
.body(format!("{}\n", url)))
@ -153,7 +164,8 @@ async fn download(
config: web::Data<Config>,
) -> Result<HttpResponse, Error> {
let id = req.match_info().query("id");
let mut rows = sqlx::query("SELECT file_id, file_name from files WHERE file_id = $1")
let mut rows =
sqlx::query("SELECT file_id, file_name, delete_on_download from files WHERE file_id = $1")
.bind(id)
.fetch(db.as_ref());
let row: PgRow = rows
@ -164,12 +176,13 @@ async fn download(
let file_id: String = row.get("file_id");
let file_name: String = row.get("file_name");
let delete_on_download: bool = row.get("delete_on_download");
let mut path = config.files_dir.clone();
path.push(&file_id);
let download = req.query_string().contains("dl");
let (content_type, mut content_disposition) = get_content_types(&path, &file_name);
if content_type.type_() == mime::TEXT && !download {
let response = if content_type.type_() == mime::TEXT && !download {
let content = fs::read_to_string(path).await.map_err(|_| {
error::ErrorInternalServerError("this file should be here but could not be found")
})?;
@ -188,7 +201,13 @@ async fn download(
.set_content_type(content_type)
.set_content_disposition(content_disposition);
file.into_response(&req)
};
if delete_on_download {
deleter::delete_by_id(&db, &file_id, &config.files_dir)
.await
.map_err(|_| error::ErrorInternalServerError("could not delete file"))?;
}
response
}
fn get_content_types(path: &Path, filename: &str) -> (Mime, ContentDisposition) {
@ -263,10 +282,12 @@ async fn setup_db() -> PgPool {
.await
.expect("could not create db pool");
sqlx::query(include_str!("../init-db.sql"))
for query in include_str!("../init-db.sql").split_inclusive(";") {
sqlx::query(query)
.execute(&pool)
.await
.expect("could not create table Files");
.expect("could not initialize database schema");
}
pool
}
@ -279,10 +300,7 @@ struct Config {
#[actix_web::main]
async fn main() -> std::io::Result<()> {
if env::var("RUST_LOG").is_err() {
env::set_var("RUST_LOG", "info");
}
env_logger::init();
env_logger::Builder::from_env(Env::default().default_filter_or("info,sqlx=warn")).init();
let pool: PgPool = setup_db().await;
let max_file_size = env::var("UPLOAD_MAX_BYTES")
@ -318,7 +336,7 @@ async fn main() -> std::io::Result<()> {
HttpServer::new({
move || {
App::new()
.wrap(middleware::Logger::default())
.wrap(middleware::Logger::new(r#"%{r}a "%r" =%s %bbytes %Tsec"#))
.app_data(db.clone())
.app_data(expiry_watch_sender.clone())
.data(config.clone())

48
src/multipart.rs
View File

@ -5,15 +5,23 @@ use async_std::{fs, fs::File, path::Path, prelude::*};
use chrono::{prelude::*, Duration};
use futures::{StreamExt, TryStreamExt};
pub(crate) struct UploadConfig {
pub original_name: String,
pub valid_till: DateTime<Local>,
pub kind: FileKind,
pub delete_on_download: bool,
}
pub(crate) async fn parse_multipart(
mut payload: Multipart,
file_id: &str,
filename: &Path,
max_size: Option<u64>,
) -> Result<(Option<String>, DateTime<Local>, FileKind), error::Error> {
) -> Result<UploadConfig, error::Error> {
let mut original_name: Option<String> = None;
let mut keep_for: Option<String> = None;
let mut kind: Option<FileKind> = None;
let mut delete_on_download = false;
while let Ok(Some(field)) = payload.try_next().await {
let name = get_field_name(&field)?;
@ -45,31 +53,41 @@ pub(crate) async fn parse_multipart(
.map_err(|_| error::ErrorInternalServerError("could not create file"))?;
write_to_file(&mut file, field, max_size).await?;
}
"delete_on_download" => {
delete_on_download = dbg!(parse_string(name, field).await?) != "false";
}
_ => {}
};
}
if let Some(original_name) = &original_name {
let original_name = original_name.ok_or_else(|| error::ErrorBadRequest("no content found"))?;
let kind = kind.ok_or_else(|| error::ErrorBadRequest("no content found"))?;
if original_name.len() > 255 {
return Err(error::ErrorBadRequest("filename is too long"));
}
}
let validity_secs = keep_for
.map(|timeout| timeout.parse())
.transpose()
.map_err(|e| error::ErrorBadRequest(format!("field validity_secs is not a number: {}", e)))?
.unwrap_or(1800); // default to 30 minutes
let max_validity_secs = Duration::days(31).num_seconds();
if validity_secs > max_validity_secs {
let valid_till = if let Some(keep_for) = keep_for {
let keep_for = keep_for.parse().map_err(|e| {
error::ErrorBadRequest(format!("field keep_for is not a number: {}", e))
})?;
let max_keep_for = Duration::days(31).num_seconds();
if keep_for > max_keep_for {
return Err(error::ErrorBadRequest(format!(
"maximum allowed validity is {} seconds, but you specified {} seconds",
max_validity_secs, validity_secs
max_keep_for, keep_for
)));
}
let valid_till = Local::now() + Duration::seconds(validity_secs);
let kind = kind.ok_or_else(|| error::ErrorBadRequest("no content found"))?;
Ok((original_name, valid_till, kind))
Local::now() + Duration::seconds(keep_for)
} else {
Local::now() + Duration::seconds(1800)
};
Ok(UploadConfig {
original_name,
valid_till,
kind,
delete_on_download,
})
}
fn get_field_name(field: &Field) -> Result<String, error::Error> {

4
static/index.css
View File

@ -48,6 +48,10 @@ textarea,
max-width: calc(100vw - 3rem - 4px);
}
input[type="checkbox"] {
margin-bottom: 1.5rem;
}
.button {
cursor: pointer;
}

7
template/index.html
View File

@ -29,6 +29,13 @@
<option value="2678400">einen monat</option>
</select>
<br />
<input
id="delete_on_download"
type="checkbox"
name="delete_on_download"
/>
<label for="delete_on_download">nach einem download löschen</label>
<br />
<input class="main button" type="submit" value="hochladen" />
</form>
<section class="usage">