2026-07-28 20:43:46 +08:00

373 lines
11 KiB
Rust

#![forbid(unsafe_code)]
use chrono::{DateTime, Utc};
use drosync_protocol::ManifestItem;
use rayon::prelude::*;
use serde::{Deserialize, Serialize};
use sha2::{Digest, Sha256};
use std::ffi::OsStr;
use std::fs::{self, File};
use std::io::{self, BufReader, Read};
use std::path::{Component, Path, PathBuf};
use std::time::{Duration, SystemTime};
use thiserror::Error;
use uuid::Uuid;
use walkdir::WalkDir;
pub const DEFAULT_STABLE_FOR: Duration = Duration::from_secs(60);
pub const DEFAULT_RENAME_SETTLE: Duration = Duration::from_secs(5);
#[derive(Debug, Error)]
pub enum CoreError {
#[error("I/O error: {0}")]
Io(#[from] io::Error),
#[error("path is outside the mapping root")]
OutsideRoot,
#[error("path contains unsupported components")]
InvalidPath,
#[error("file changed while hashing")]
ChangedDuringHash,
#[error("scan page size must be greater than zero")]
InvalidPageSize,
#[error("scan page sink failed: {0}")]
Sink(String),
}
#[derive(Debug, Clone, Serialize, Deserialize)]
#[serde(rename_all = "camelCase")]
pub struct Mapping {
pub id: String,
pub local_root: PathBuf,
pub sync_space_id: String,
#[serde(default)]
pub remote_prefix: String,
#[serde(default)]
pub include_hidden: bool,
#[serde(default)]
pub require_done_marker: bool,
}
impl Mapping {
#[must_use]
pub fn new(local_root: PathBuf, sync_space_id: String) -> Self {
Self {
id: Uuid::new_v4().to_string(),
local_root,
sync_space_id,
remote_prefix: String::new(),
include_hidden: false,
require_done_marker: false,
}
}
}
#[derive(Debug, Clone)]
pub struct ScanOptions {
pub include_hidden: bool,
pub require_done_marker: bool,
pub stable_for: Duration,
}
impl Default for ScanOptions {
fn default() -> Self {
Self {
include_hidden: false,
require_done_marker: false,
stable_for: DEFAULT_STABLE_FOR,
}
}
}
#[derive(Debug, Clone)]
pub struct ScannedFile {
pub absolute_path: PathBuf,
pub manifest: ManifestItem,
}
#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
pub struct ScanSummary {
pub file_count: u64,
pub total_bytes: i64,
}
const TEMP_SUFFIXES: &[&str] = &["tmp", "part", "partial", "crdownload", "swp", "download"];
fn is_hidden(path: &Path) -> bool {
path.file_name()
.and_then(OsStr::to_str)
.is_some_and(|name| name.starts_with('.'))
}
fn is_temporary(path: &Path) -> bool {
path.extension()
.and_then(OsStr::to_str)
.is_some_and(|value| {
TEMP_SUFFIXES
.iter()
.any(|suffix| value.eq_ignore_ascii_case(suffix))
})
}
fn normalized_relative(root: &Path, path: &Path) -> Result<String, CoreError> {
let relative = path
.strip_prefix(root)
.map_err(|_| CoreError::OutsideRoot)?;
let mut parts = Vec::new();
for component in relative.components() {
match component {
Component::Normal(value) => {
let value = value.to_str().ok_or(CoreError::InvalidPath)?;
if value.is_empty()
|| value == "."
|| value == ".."
|| value.chars().any(char::is_control)
{
return Err(CoreError::InvalidPath);
}
parts.push(value);
}
_ => return Err(CoreError::InvalidPath),
}
}
if parts.is_empty() {
return Err(CoreError::InvalidPath);
}
Ok(parts.join("/"))
}
fn hash_stable_file(path: &Path) -> Result<(String, fs::Metadata), CoreError> {
let before = fs::metadata(path)?;
let mut reader = BufReader::new(File::open(path)?);
let mut hasher = Sha256::new();
let mut buffer = vec![0_u8; 1024 * 1024];
loop {
let read = reader.read(&mut buffer)?;
if read == 0 {
break;
}
hasher.update(&buffer[..read]);
}
let after = fs::metadata(path)?;
if before.len() != after.len() || before.modified()? != after.modified()? {
return Err(CoreError::ChangedDuringHash);
}
Ok((hex_digest(hasher.finalize().as_slice()), after))
}
fn hex_digest(bytes: &[u8]) -> String {
const HEX: &[u8; 16] = b"0123456789abcdef";
let mut output = String::with_capacity(bytes.len() * 2);
for byte in bytes {
output.push(char::from(HEX[usize::from(byte >> 4)]));
output.push(char::from(HEX[usize::from(byte & 0x0f)]));
}
output
}
fn hash_scan_batch(
pool: &rayon::ThreadPool,
canonical_root: &Path,
candidates: &[PathBuf],
) -> Result<Vec<ScannedFile>, CoreError> {
let mut output = pool.install(|| {
candidates
.par_iter()
.map(|path| {
let relative_path = normalized_relative(canonical_root, path)?;
let (sha256, stable_metadata) = hash_stable_file(path)?;
let modified_at: DateTime<Utc> = DateTime::from(stable_metadata.modified()?);
Ok(ScannedFile {
absolute_path: path.clone(),
manifest: ManifestItem {
client_item_id: Uuid::new_v4().to_string(),
relative_path,
kind: "file".to_owned(),
size_bytes: i64::try_from(stable_metadata.len()).unwrap_or(i64::MAX),
modified_at: modified_at.to_rfc3339(),
sha256,
content_type: "application/octet-stream".to_owned(),
last_known_remote: drosync_protocol::RemoteBase::default(),
},
})
})
.collect::<Result<Vec<_>, CoreError>>()
})?;
output.sort_by(|left, right| {
left.manifest
.relative_path
.cmp(&right.manifest.relative_path)
});
Ok(output)
}
pub fn scan_mapping_paged<F>(
mapping: &Mapping,
options: &ScanOptions,
page_size: usize,
mut emit: F,
) -> Result<ScanSummary, CoreError>
where
F: FnMut(Vec<ScannedFile>) -> Result<(), CoreError>,
{
if page_size == 0 {
return Err(CoreError::InvalidPageSize);
}
let canonical_root = mapping.local_root.canonicalize()?;
let worker_count = std::thread::available_parallelism()
.map_or(2, usize::from)
.clamp(1, 4);
let pool = rayon::ThreadPoolBuilder::new()
.num_threads(worker_count)
.thread_name(|index| format!("drosync-hash-{index}"))
.build()
.map_err(io::Error::other)?;
let mut candidates = Vec::with_capacity(page_size);
let mut summary = ScanSummary::default();
for entry in WalkDir::new(&canonical_root)
.follow_links(false)
.into_iter()
.filter_entry(|entry| {
options.include_hidden || entry.path() == canonical_root || !is_hidden(entry.path())
})
.filter_map(Result::ok)
{
let path = entry.path();
if path == canonical_root {
continue;
}
if (!options.include_hidden && is_hidden(path)) || entry.file_type().is_symlink() {
continue;
}
if !entry.file_type().is_file() || is_temporary(path) {
continue;
}
if options.require_done_marker {
let marker = PathBuf::from(format!("{}.done", path.to_string_lossy()));
if !marker.is_file() {
continue;
}
}
let metadata = entry.metadata().map_err(io::Error::other)?;
let modified = metadata.modified()?;
if SystemTime::now()
.duration_since(modified)
.unwrap_or_default()
< options.stable_for
{
continue;
}
candidates.push(path.to_path_buf());
if candidates.len() == page_size {
let page = hash_scan_batch(&pool, &canonical_root, &candidates)?;
summary.file_count = summary
.file_count
.saturating_add(u64::try_from(page.len()).unwrap_or(u64::MAX));
summary.total_bytes = page.iter().fold(summary.total_bytes, |total, file| {
total.saturating_add(file.manifest.size_bytes)
});
emit(page)?;
candidates.clear();
}
}
if !candidates.is_empty() {
let page = hash_scan_batch(&pool, &canonical_root, &candidates)?;
summary.file_count = summary
.file_count
.saturating_add(u64::try_from(page.len()).unwrap_or(u64::MAX));
summary.total_bytes = page.iter().fold(summary.total_bytes, |total, file| {
total.saturating_add(file.manifest.size_bytes)
});
emit(page)?;
}
Ok(summary)
}
pub fn scan_mapping(
mapping: &Mapping,
options: &ScanOptions,
) -> Result<Vec<ScannedFile>, CoreError> {
let mut output = Vec::new();
scan_mapping_paged(mapping, options, 1_000, |mut page| {
output.append(&mut page);
Ok(())
})?;
output.sort_by(|left, right| {
left.manifest
.relative_path
.cmp(&right.manifest.relative_path)
});
Ok(output)
}
#[cfg(test)]
mod tests {
use super::{
CoreError, Mapping, ScanOptions, normalized_relative, scan_mapping, scan_mapping_paged,
};
use std::fs;
use std::path::Path;
use std::time::Duration;
#[test]
fn scan_skips_hidden_symlink_and_temporary_files() {
let root = tempfile::tempdir().unwrap();
fs::write(root.path().join("accepted.bin"), b"data").unwrap();
fs::write(root.path().join(".hidden"), b"hidden").unwrap();
fs::write(root.path().join("upload.part"), b"partial").unwrap();
let mapping = Mapping::new(root.path().to_path_buf(), "space".to_owned());
let files = scan_mapping(
&mapping,
&ScanOptions {
stable_for: Duration::ZERO,
..ScanOptions::default()
},
)
.unwrap();
assert_eq!(files.len(), 1);
assert_eq!(files[0].manifest.relative_path, "accepted.bin");
}
#[test]
fn paged_scan_bounds_each_emitted_batch() {
let root = tempfile::tempdir().unwrap();
for index in 0..5 {
fs::write(root.path().join(format!("{index}.bin")), b"data").unwrap();
}
let mapping = Mapping::new(root.path().to_path_buf(), "space".to_owned());
let mut page_lengths = Vec::new();
let summary = scan_mapping_paged(
&mapping,
&ScanOptions {
stable_for: Duration::ZERO,
..ScanOptions::default()
},
2,
|page| {
page_lengths.push(page.len());
Ok(())
},
)
.unwrap();
assert_eq!(page_lengths, vec![2, 2, 1]);
assert_eq!(summary.file_count, 5);
assert_eq!(summary.total_bytes, 20);
}
#[test]
fn relative_path_rejects_escape_and_control_characters() {
let root = Path::new("/mapping");
assert!(matches!(
normalized_relative(root, Path::new("/mapping/../outside.bin")),
Err(CoreError::InvalidPath)
));
assert!(matches!(
normalized_relative(root, Path::new("/mapping/bad\nname.bin")),
Err(CoreError::InvalidPath)
));
assert_eq!(
normalized_relative(root, Path::new("/mapping/nested/good.bin")).unwrap(),
"nested/good.bin"
);
}
}