From b4228aa74f6ef4720167236cb072b84d94aa6d2a Mon Sep 17 00:00:00 2001 From: murilo ijanc Date: Fri, 27 Mar 2026 21:54:25 -0300 Subject: Add chunked paste support for content up to 1.44 MB Large pastes are split into 8 KiB chunks on the client side, each stored separately in a dedicated chunks/ directory. A version-2 manifest paste lists the chunk hashes and is announced to the DHT; chunks replicate via periodic republish with per-put throttling to avoid rate-limit bans. - New PUTC/PUTM protocol commands for chunks and manifests - Client-side chunking avoids O(n^2) base58 on large content - HTTP handler reassembles chunks directly from store - DHT sync routes incoming chunks to chunks/ directory - Republish interval reduced to 5 min with 200ms throttle - tp.1 updated with new 1.44 MB limit --- src/bin/tp.rs | 323 ++++++++++++++++++++++++++++++++++++++++++++++++++-------- 1 file changed, 282 insertions(+), 41 deletions(-) (limited to 'src/bin') diff --git a/src/bin/tp.rs b/src/bin/tp.rs index fbfc872..9a69832 100644 --- a/src/bin/tp.rs +++ b/src/bin/tp.rs @@ -2,32 +2,84 @@ //! //! Sends commands to the `tpd` daemon over a Unix socket. //! Reads paste content from stdin (put) and writes it to -//! stdout (get). +//! stdout (get). Large pastes (> 8 KiB) are automatically +//! split into chunks on the client side. +use std::collections::BTreeMap; use std::io::{BufRead, BufReader, Read, Write}; use std::os::unix::net::UnixStream; use std::path::PathBuf; #[path = "../base58.rs"] mod base58; +#[path = "../crypto.rs"] +mod crypto; #[path = "../sandbox.rs"] mod sandbox; +/// Maximum paste size: 1.44 MB (floppy disk). +const MAX_PASTE: usize = 1_440 * 1024; + +/// Chunk size matching the DHT fragment limit. +const CHUNK_SIZE: usize = 8 * 1024; + fn default_socket() -> PathBuf { PathBuf::from("/var/tesseras-paste/daemon.sock") } +fn labels_path() -> PathBuf { + if let Ok(home) = std::env::var("HOME") { + PathBuf::from(home).join(".config/tp/labels") + } else { + PathBuf::from("/tmp/tp-labels") + } +} + +fn load_labels(path: &PathBuf) -> BTreeMap { + let mut map = BTreeMap::new(); + let data = match std::fs::read_to_string(path) { + Ok(d) => d, + Err(_) => return map, + }; + for line in data.lines() { + if let Some((key, label)) = line.split_once('\t') { + map.insert(key.to_string(), label.to_string()); + } + } + map +} + +fn save_labels(path: &PathBuf, labels: &BTreeMap) { + if let Some(parent) = path.parent() { + let _ = std::fs::create_dir_all(parent); + } + let mut buf = String::new(); + for (key, label) in labels { + buf.push_str(key); + buf.push('\t'); + buf.push_str(label); + buf.push('\n'); + } + if let Err(e) = std::fs::write(path, buf.as_bytes()) { + eprintln!("warning: could not save labels: {e}"); + } +} + fn usage() { eprintln!("usage: tp [-s sock] [-v] [args]"); eprintln!(); eprintln!("commands:"); - eprintln!(" put [-t ttl] [-p] read stdin, store paste"); + eprintln!(" put [-t ttl] [-p] [-l label]"); + eprintln!(" read stdin, store paste"); eprintln!(" -p public (no encryption)"); + eprintln!(" -l attach a label"); eprintln!(" get retrieve paste to stdout"); - eprintln!(" del delete paste"); - eprintln!(" pin pin (never expires)"); - eprintln!(" unpin unpin"); - eprintln!(" status show daemon status"); + eprintln!(" del delete paste"); + eprintln!(" pin pin (never expires)"); + eprintln!(" unpin unpin"); + eprintln!(" list list labeled pastes"); + eprintln!(" label add or update a label"); + eprintln!(" status show daemon status"); eprintln!(); eprintln!(" -s sock Unix socket path"); eprintln!(" -v verbose output"); @@ -49,6 +101,47 @@ fn parse_ttl(s: &str) -> Result { } } +/// Send a request over the socket and read the response. +/// Returns the data on OK, or exits on ERR. +fn send_recv( + stream: &UnixStream, + reader: &mut BufReader<&UnixStream>, + request: &str, + verbose: bool, +) -> String { + if verbose { + eprintln!(">> {}", request.trim()); + } + if let Err(e) = (stream as &UnixStream).write_all(request.as_bytes()) { + eprintln!("error: writing to socket: {e}"); + std::process::exit(1); + } + let mut line = String::new(); + match reader.read_line(&mut line) { + Ok(0) => { + eprintln!("error: daemon closed connection"); + std::process::exit(1); + } + Err(e) => { + eprintln!("error: reading from socket: {e}"); + std::process::exit(1); + } + _ => {} + } + if verbose { + eprintln!("<< {}", line.trim()); + } + if let Some(data) = line.trim().strip_prefix("OK ") { + data.to_string() + } else if let Some(msg) = line.trim().strip_prefix("ERR ") { + eprintln!("error: {msg}"); + std::process::exit(1); + } else { + eprintln!("error: unexpected response: {}", line.trim()); + std::process::exit(1); + } +} + fn main() { let args: Vec = std::env::args().collect(); @@ -89,12 +182,62 @@ fn main() { } let command = &cmd_args[0]; + + // Handle client-only commands before sandboxing + match command.as_str() { + "list" => { + let lpath = labels_path(); + let labels = load_labels(&lpath); + if labels.is_empty() { + eprintln!("no labels"); + } else { + for (key, label) in &labels { + println!("{key}\t{label}"); + } + } + return; + } + "label" => { + let key = cmd_args.get(1).unwrap_or_else(|| { + eprintln!("error: label requires a key and text"); + std::process::exit(1); + }); + let text: String = cmd_args[2..].join(" "); + if text.is_empty() { + eprintln!("error: label requires text"); + std::process::exit(1); + } + let lpath = labels_path(); + let mut labels = load_labels(&lpath); + labels.insert(key.clone(), text); + save_labels(&lpath, &labels); + return; + } + _ => {} + } + + // ── Parse command into request(s) ────────────────── + + // For most commands we build a single request string. + // The "put" command may use chunked mode (multiple requests). + enum PutData { + Single(String), + Chunked { + data: Vec, + ttl_secs: u64, + enc_key: Option<[u8; crypto::KEY_SIZE]>, + }, + } + let mut is_get = false; + let mut put_label: Option = None; + let mut del_key: Option = None; - let request = match command.as_str() { + let put_data = match command.as_str() { "put" => { let mut ttl = "24h".to_string(); let mut public = false; + let mut label: Option = None; let mut j = 1; while j < cmd_args.len() { match cmd_args[j].as_str() { @@ -105,6 +248,12 @@ fn main() { } } "-p" => public = true, + "-l" => { + j += 1; + if j < cmd_args.len() { + label = Some(cmd_args[j].clone()); + } + } _ => {} } j += 1; @@ -116,9 +265,7 @@ fn main() { std::process::exit(1); } }; - // Read at most MAX_PASTE + 1 byte so we can detect - // oversized input without unbounded allocation. - const MAX_PASTE: usize = 64 * 1024; + let mut content = Vec::new(); match std::io::stdin() .take((MAX_PASTE + 1) as u64) @@ -129,7 +276,7 @@ fn main() { std::process::exit(1); } Ok(n) if n > MAX_PASTE => { - eprintln!("error: input exceeds 64 KiB limit"); + eprintln!("error: input exceeds 1.44 MB limit"); std::process::exit(1); } Err(e) => { @@ -138,8 +285,31 @@ fn main() { } _ => {} } - let cmd = if public { "PUTP" } else { "PUT" }; - format!("{cmd} {ttl_secs} {}\n", base58::encode(&content)) + + put_label = label; + + if content.len() <= CHUNK_SIZE { + // Small paste — single PUT/PUTP + let cmd = if public { "PUTP" } else { "PUT" }; + PutData::Single(format!( + "{cmd} {ttl_secs} {}\n", + base58::encode(&content) + )) + } else { + // Large paste — client-side chunking + let (data, enc_key) = if public { + (content, None) + } else { + let key = crypto::generate_key(); + let encrypted = crypto::encrypt(&key, &content); + (encrypted, Some(key)) + }; + PutData::Chunked { + data, + ttl_secs, + enc_key, + } + } } "get" => { let key = cmd_args.get(1).unwrap_or_else(|| { @@ -147,30 +317,31 @@ fn main() { std::process::exit(1); }); is_get = true; - format!("GET {key}\n") + PutData::Single(format!("GET {key}\n")) } "del" => { let key = cmd_args.get(1).unwrap_or_else(|| { eprintln!("error: del requires a key"); std::process::exit(1); }); - format!("DEL {key}\n") + del_key = Some(key.clone()); + PutData::Single(format!("DEL {key}\n")) } "pin" => { let key = cmd_args.get(1).unwrap_or_else(|| { eprintln!("error: pin requires a key"); std::process::exit(1); }); - format!("PIN {key}\n") + PutData::Single(format!("PIN {key}\n")) } "unpin" => { let key = cmd_args.get(1).unwrap_or_else(|| { eprintln!("error: unpin requires a key"); std::process::exit(1); }); - format!("UNPIN {key}\n") + PutData::Single(format!("UNPIN {key}\n")) } - "status" => "STATUS\n".to_string(), + "status" => PutData::Single("STATUS\n".to_string()), other => { eprintln!("unknown command: {other}"); usage(); @@ -179,19 +350,23 @@ fn main() { }; // ── Sandbox ───────────────────────────────────── + let lpath = labels_path(); sandbox::do_unveil(&sock_path, "rw"); + if let Some(parent) = lpath.parent() { + let _ = std::fs::create_dir_all(parent); + sandbox::do_unveil(parent, "rwc"); + } sandbox::unveil_lock(); - sandbox::do_pledge("stdio unix rpath"); + sandbox::do_pledge("stdio unix rpath wpath cpath"); if verbose { eprintln!("socket: {}", sock_path.display()); - eprintln!(">> {}", request.trim()); } let stream = match UnixStream::connect(&sock_path) { Ok(s) => s, Err(e) => { - eprintln!("error: cannot connect to {}: {e}", sock_path.display(),); + eprintln!("error: cannot connect to {}: {e}", sock_path.display()); eprintln!("hint: is tpd running?"); std::process::exit(1); } @@ -201,25 +376,81 @@ fn main() { .set_read_timeout(Some(std::time::Duration::from_secs(60))) .ok(); - let mut writer = &stream; - if let Err(e) = writer.write_all(request.as_bytes()) { - eprintln!("error: writing to socket: {e}"); - std::process::exit(1); - } + let mut reader = BufReader::new(&stream); + + match put_data { + PutData::Chunked { + data, + ttl_secs, + enc_key, + } => { + // ── Chunked put ───────────────────────── + let n_chunks = + (data.len() + CHUNK_SIZE - 1) / CHUNK_SIZE; + if verbose { + eprintln!( + "chunked: {} bytes, {} chunks", + data.len(), + n_chunks, + ); + } + + // Send each chunk via PUTC + let mut chunk_hashes: Vec> = Vec::new(); + for (i, chunk) in data.chunks(CHUNK_SIZE).enumerate() { + let req = format!( + "PUTC {} {}\n", + ttl_secs, + base58::encode(chunk), + ); + let hash_b58 = send_recv(&stream, &mut reader, &req, verbose); + let hash = base58::decode(&hash_b58).unwrap_or_else(|| { + eprintln!("error: invalid hash from daemon"); + std::process::exit(1); + }); + if verbose { + eprintln!("chunk {}/{}: {hash_b58}", i + 1, n_chunks); + } + chunk_hashes.push(hash); + } + + // Build manifest: count(u16 BE) || hash1 || hash2 || ... + let count = chunk_hashes.len() as u16; + let mut manifest = + Vec::with_capacity(2 + 32 * chunk_hashes.len()); + manifest.extend_from_slice(&count.to_be_bytes()); + for hash in &chunk_hashes { + manifest.extend_from_slice(hash); + } + + let req = format!( + "PUTM {} {}\n", + ttl_secs, + base58::encode(&manifest), + ); + let manifest_hash = send_recv(&stream, &mut reader, &req, verbose); - let reader = BufReader::new(&stream); - for line in reader.lines() { - let line = match line { - Ok(l) => l, - Err(_) => break, - }; - if verbose { - eprintln!("<< {}", line); + let key_str = match enc_key { + Some(key) => { + format!("{manifest_hash}#{}", base58::encode(&key)) + } + None => manifest_hash, + }; + println!("{key_str}"); + + // Save label + if let Some(ref label) = put_label { + let mut labels = load_labels(&lpath); + labels.insert(key_str, label.clone()); + save_labels(&lpath, &labels); + } } - if let Some(data) = line.strip_prefix("OK ") { + PutData::Single(request) => { + // ── Single request ────────────────────── + let data = send_recv(&stream, &mut reader, &request, verbose); + if is_get { - // Decode base58 → raw bytes → stdout - match base58::decode(data) { + match base58::decode(&data) { Some(bytes) => { if let Err(e) = std::io::stdout().write_all(&bytes) { eprintln!("error: writing to stdout: {e}"); @@ -231,10 +462,20 @@ fn main() { } else { println!("{data}"); } - break; - } else if let Some(msg) = line.strip_prefix("ERR ") { - eprintln!("error: {msg}"); - std::process::exit(1); + + // Save label on successful put + if let Some(ref label) = put_label { + let mut labels = load_labels(&lpath); + labels.insert(data.to_string(), label.clone()); + save_labels(&lpath, &labels); + } + // Remove label on successful del + if let Some(ref key) = del_key { + let mut labels = load_labels(&lpath); + if labels.remove(key).is_some() { + save_labels(&lpath, &labels); + } + } } } } -- cgit v1.2.3