373 lines
11 KiB
Rust
373 lines
11 KiB
Rust
#![forbid(unsafe_code)]
|
|
|
|
use chrono::{DateTime, Utc};
|
|
use drosync_protocol::ManifestItem;
|
|
use rayon::prelude::*;
|
|
use serde::{Deserialize, Serialize};
|
|
use sha2::{Digest, Sha256};
|
|
use std::ffi::OsStr;
|
|
use std::fs::{self, File};
|
|
use std::io::{self, BufReader, Read};
|
|
use std::path::{Component, Path, PathBuf};
|
|
use std::time::{Duration, SystemTime};
|
|
use thiserror::Error;
|
|
use uuid::Uuid;
|
|
use walkdir::WalkDir;
|
|
|
|
pub const DEFAULT_STABLE_FOR: Duration = Duration::from_secs(60);
|
|
pub const DEFAULT_RENAME_SETTLE: Duration = Duration::from_secs(5);
|
|
|
|
#[derive(Debug, Error)]
|
|
pub enum CoreError {
|
|
#[error("I/O error: {0}")]
|
|
Io(#[from] io::Error),
|
|
#[error("path is outside the mapping root")]
|
|
OutsideRoot,
|
|
#[error("path contains unsupported components")]
|
|
InvalidPath,
|
|
#[error("file changed while hashing")]
|
|
ChangedDuringHash,
|
|
#[error("scan page size must be greater than zero")]
|
|
InvalidPageSize,
|
|
#[error("scan page sink failed: {0}")]
|
|
Sink(String),
|
|
}
|
|
|
|
#[derive(Debug, Clone, Serialize, Deserialize)]
|
|
#[serde(rename_all = "camelCase")]
|
|
pub struct Mapping {
|
|
pub id: String,
|
|
pub local_root: PathBuf,
|
|
pub sync_space_id: String,
|
|
#[serde(default)]
|
|
pub remote_prefix: String,
|
|
#[serde(default)]
|
|
pub include_hidden: bool,
|
|
#[serde(default)]
|
|
pub require_done_marker: bool,
|
|
}
|
|
|
|
impl Mapping {
|
|
#[must_use]
|
|
pub fn new(local_root: PathBuf, sync_space_id: String) -> Self {
|
|
Self {
|
|
id: Uuid::new_v4().to_string(),
|
|
local_root,
|
|
sync_space_id,
|
|
remote_prefix: String::new(),
|
|
include_hidden: false,
|
|
require_done_marker: false,
|
|
}
|
|
}
|
|
}
|
|
|
|
#[derive(Debug, Clone)]
|
|
pub struct ScanOptions {
|
|
pub include_hidden: bool,
|
|
pub require_done_marker: bool,
|
|
pub stable_for: Duration,
|
|
}
|
|
|
|
impl Default for ScanOptions {
|
|
fn default() -> Self {
|
|
Self {
|
|
include_hidden: false,
|
|
require_done_marker: false,
|
|
stable_for: DEFAULT_STABLE_FOR,
|
|
}
|
|
}
|
|
}
|
|
|
|
#[derive(Debug, Clone)]
|
|
pub struct ScannedFile {
|
|
pub absolute_path: PathBuf,
|
|
pub manifest: ManifestItem,
|
|
}
|
|
|
|
#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
|
|
pub struct ScanSummary {
|
|
pub file_count: u64,
|
|
pub total_bytes: i64,
|
|
}
|
|
|
|
const TEMP_SUFFIXES: &[&str] = &["tmp", "part", "partial", "crdownload", "swp", "download"];
|
|
|
|
fn is_hidden(path: &Path) -> bool {
|
|
path.file_name()
|
|
.and_then(OsStr::to_str)
|
|
.is_some_and(|name| name.starts_with('.'))
|
|
}
|
|
|
|
fn is_temporary(path: &Path) -> bool {
|
|
path.extension()
|
|
.and_then(OsStr::to_str)
|
|
.is_some_and(|value| {
|
|
TEMP_SUFFIXES
|
|
.iter()
|
|
.any(|suffix| value.eq_ignore_ascii_case(suffix))
|
|
})
|
|
}
|
|
|
|
fn normalized_relative(root: &Path, path: &Path) -> Result<String, CoreError> {
|
|
let relative = path
|
|
.strip_prefix(root)
|
|
.map_err(|_| CoreError::OutsideRoot)?;
|
|
let mut parts = Vec::new();
|
|
for component in relative.components() {
|
|
match component {
|
|
Component::Normal(value) => {
|
|
let value = value.to_str().ok_or(CoreError::InvalidPath)?;
|
|
if value.is_empty()
|
|
|| value == "."
|
|
|| value == ".."
|
|
|| value.chars().any(char::is_control)
|
|
{
|
|
return Err(CoreError::InvalidPath);
|
|
}
|
|
parts.push(value);
|
|
}
|
|
_ => return Err(CoreError::InvalidPath),
|
|
}
|
|
}
|
|
if parts.is_empty() {
|
|
return Err(CoreError::InvalidPath);
|
|
}
|
|
Ok(parts.join("/"))
|
|
}
|
|
|
|
fn hash_stable_file(path: &Path) -> Result<(String, fs::Metadata), CoreError> {
|
|
let before = fs::metadata(path)?;
|
|
let mut reader = BufReader::new(File::open(path)?);
|
|
let mut hasher = Sha256::new();
|
|
let mut buffer = vec![0_u8; 1024 * 1024];
|
|
loop {
|
|
let read = reader.read(&mut buffer)?;
|
|
if read == 0 {
|
|
break;
|
|
}
|
|
hasher.update(&buffer[..read]);
|
|
}
|
|
let after = fs::metadata(path)?;
|
|
if before.len() != after.len() || before.modified()? != after.modified()? {
|
|
return Err(CoreError::ChangedDuringHash);
|
|
}
|
|
Ok((hex_digest(hasher.finalize().as_slice()), after))
|
|
}
|
|
|
|
fn hex_digest(bytes: &[u8]) -> String {
|
|
const HEX: &[u8; 16] = b"0123456789abcdef";
|
|
let mut output = String::with_capacity(bytes.len() * 2);
|
|
for byte in bytes {
|
|
output.push(char::from(HEX[usize::from(byte >> 4)]));
|
|
output.push(char::from(HEX[usize::from(byte & 0x0f)]));
|
|
}
|
|
output
|
|
}
|
|
|
|
fn hash_scan_batch(
|
|
pool: &rayon::ThreadPool,
|
|
canonical_root: &Path,
|
|
candidates: &[PathBuf],
|
|
) -> Result<Vec<ScannedFile>, CoreError> {
|
|
let mut output = pool.install(|| {
|
|
candidates
|
|
.par_iter()
|
|
.map(|path| {
|
|
let relative_path = normalized_relative(canonical_root, path)?;
|
|
let (sha256, stable_metadata) = hash_stable_file(path)?;
|
|
let modified_at: DateTime<Utc> = DateTime::from(stable_metadata.modified()?);
|
|
Ok(ScannedFile {
|
|
absolute_path: path.clone(),
|
|
manifest: ManifestItem {
|
|
client_item_id: Uuid::new_v4().to_string(),
|
|
relative_path,
|
|
kind: "file".to_owned(),
|
|
size_bytes: i64::try_from(stable_metadata.len()).unwrap_or(i64::MAX),
|
|
modified_at: modified_at.to_rfc3339(),
|
|
sha256,
|
|
content_type: "application/octet-stream".to_owned(),
|
|
last_known_remote: drosync_protocol::RemoteBase::default(),
|
|
},
|
|
})
|
|
})
|
|
.collect::<Result<Vec<_>, CoreError>>()
|
|
})?;
|
|
output.sort_by(|left, right| {
|
|
left.manifest
|
|
.relative_path
|
|
.cmp(&right.manifest.relative_path)
|
|
});
|
|
Ok(output)
|
|
}
|
|
|
|
pub fn scan_mapping_paged<F>(
|
|
mapping: &Mapping,
|
|
options: &ScanOptions,
|
|
page_size: usize,
|
|
mut emit: F,
|
|
) -> Result<ScanSummary, CoreError>
|
|
where
|
|
F: FnMut(Vec<ScannedFile>) -> Result<(), CoreError>,
|
|
{
|
|
if page_size == 0 {
|
|
return Err(CoreError::InvalidPageSize);
|
|
}
|
|
let canonical_root = mapping.local_root.canonicalize()?;
|
|
let worker_count = std::thread::available_parallelism()
|
|
.map_or(2, usize::from)
|
|
.clamp(1, 4);
|
|
let pool = rayon::ThreadPoolBuilder::new()
|
|
.num_threads(worker_count)
|
|
.thread_name(|index| format!("drosync-hash-{index}"))
|
|
.build()
|
|
.map_err(io::Error::other)?;
|
|
let mut candidates = Vec::with_capacity(page_size);
|
|
let mut summary = ScanSummary::default();
|
|
for entry in WalkDir::new(&canonical_root)
|
|
.follow_links(false)
|
|
.into_iter()
|
|
.filter_entry(|entry| {
|
|
options.include_hidden || entry.path() == canonical_root || !is_hidden(entry.path())
|
|
})
|
|
.filter_map(Result::ok)
|
|
{
|
|
let path = entry.path();
|
|
if path == canonical_root {
|
|
continue;
|
|
}
|
|
if (!options.include_hidden && is_hidden(path)) || entry.file_type().is_symlink() {
|
|
continue;
|
|
}
|
|
if !entry.file_type().is_file() || is_temporary(path) {
|
|
continue;
|
|
}
|
|
if options.require_done_marker {
|
|
let marker = PathBuf::from(format!("{}.done", path.to_string_lossy()));
|
|
if !marker.is_file() {
|
|
continue;
|
|
}
|
|
}
|
|
let metadata = entry.metadata().map_err(io::Error::other)?;
|
|
let modified = metadata.modified()?;
|
|
if SystemTime::now()
|
|
.duration_since(modified)
|
|
.unwrap_or_default()
|
|
< options.stable_for
|
|
{
|
|
continue;
|
|
}
|
|
candidates.push(path.to_path_buf());
|
|
if candidates.len() == page_size {
|
|
let page = hash_scan_batch(&pool, &canonical_root, &candidates)?;
|
|
summary.file_count = summary
|
|
.file_count
|
|
.saturating_add(u64::try_from(page.len()).unwrap_or(u64::MAX));
|
|
summary.total_bytes = page.iter().fold(summary.total_bytes, |total, file| {
|
|
total.saturating_add(file.manifest.size_bytes)
|
|
});
|
|
emit(page)?;
|
|
candidates.clear();
|
|
}
|
|
}
|
|
if !candidates.is_empty() {
|
|
let page = hash_scan_batch(&pool, &canonical_root, &candidates)?;
|
|
summary.file_count = summary
|
|
.file_count
|
|
.saturating_add(u64::try_from(page.len()).unwrap_or(u64::MAX));
|
|
summary.total_bytes = page.iter().fold(summary.total_bytes, |total, file| {
|
|
total.saturating_add(file.manifest.size_bytes)
|
|
});
|
|
emit(page)?;
|
|
}
|
|
Ok(summary)
|
|
}
|
|
|
|
pub fn scan_mapping(
|
|
mapping: &Mapping,
|
|
options: &ScanOptions,
|
|
) -> Result<Vec<ScannedFile>, CoreError> {
|
|
let mut output = Vec::new();
|
|
scan_mapping_paged(mapping, options, 1_000, |mut page| {
|
|
output.append(&mut page);
|
|
Ok(())
|
|
})?;
|
|
output.sort_by(|left, right| {
|
|
left.manifest
|
|
.relative_path
|
|
.cmp(&right.manifest.relative_path)
|
|
});
|
|
Ok(output)
|
|
}
|
|
|
|
#[cfg(test)]
|
|
mod tests {
|
|
use super::{
|
|
CoreError, Mapping, ScanOptions, normalized_relative, scan_mapping, scan_mapping_paged,
|
|
};
|
|
use std::fs;
|
|
use std::path::Path;
|
|
use std::time::Duration;
|
|
|
|
#[test]
|
|
fn scan_skips_hidden_symlink_and_temporary_files() {
|
|
let root = tempfile::tempdir().unwrap();
|
|
fs::write(root.path().join("accepted.bin"), b"data").unwrap();
|
|
fs::write(root.path().join(".hidden"), b"hidden").unwrap();
|
|
fs::write(root.path().join("upload.part"), b"partial").unwrap();
|
|
let mapping = Mapping::new(root.path().to_path_buf(), "space".to_owned());
|
|
let files = scan_mapping(
|
|
&mapping,
|
|
&ScanOptions {
|
|
stable_for: Duration::ZERO,
|
|
..ScanOptions::default()
|
|
},
|
|
)
|
|
.unwrap();
|
|
assert_eq!(files.len(), 1);
|
|
assert_eq!(files[0].manifest.relative_path, "accepted.bin");
|
|
}
|
|
|
|
#[test]
|
|
fn paged_scan_bounds_each_emitted_batch() {
|
|
let root = tempfile::tempdir().unwrap();
|
|
for index in 0..5 {
|
|
fs::write(root.path().join(format!("{index}.bin")), b"data").unwrap();
|
|
}
|
|
let mapping = Mapping::new(root.path().to_path_buf(), "space".to_owned());
|
|
let mut page_lengths = Vec::new();
|
|
let summary = scan_mapping_paged(
|
|
&mapping,
|
|
&ScanOptions {
|
|
stable_for: Duration::ZERO,
|
|
..ScanOptions::default()
|
|
},
|
|
2,
|
|
|page| {
|
|
page_lengths.push(page.len());
|
|
Ok(())
|
|
},
|
|
)
|
|
.unwrap();
|
|
assert_eq!(page_lengths, vec![2, 2, 1]);
|
|
assert_eq!(summary.file_count, 5);
|
|
assert_eq!(summary.total_bytes, 20);
|
|
}
|
|
|
|
#[test]
|
|
fn relative_path_rejects_escape_and_control_characters() {
|
|
let root = Path::new("/mapping");
|
|
assert!(matches!(
|
|
normalized_relative(root, Path::new("/mapping/../outside.bin")),
|
|
Err(CoreError::InvalidPath)
|
|
));
|
|
assert!(matches!(
|
|
normalized_relative(root, Path::new("/mapping/bad\nname.bin")),
|
|
Err(CoreError::InvalidPath)
|
|
));
|
|
assert_eq!(
|
|
normalized_relative(root, Path::new("/mapping/nested/good.bin")).unwrap(),
|
|
"nested/good.bin"
|
|
);
|
|
}
|
|
}
|