From 7ff6fd9a75b525f2e915025e9378fc3d445a52be Mon Sep 17 00:00:00 2001 From: first Date: Fri, 24 Jan 2025 07:17:34 +0000 Subject: [PATCH] re-add burn and create-iso Signed-off-by: first --- backup2mdisc.sh | 296 ++++++++++++++++++++++++++++++++---------------- 1 file changed, 199 insertions(+), 97 deletions(-) diff --git a/backup2mdisc.sh b/backup2mdisc.sh index bc41de8..9891279 100644 --- a/backup2mdisc.sh +++ b/backup2mdisc.sh @@ -3,175 +3,277 @@ # backup2mdisc.sh # # Purpose: -# Creates multiple self-contained 100GB (default) backup archives, each encrypted -# independently. Useful for writing to large-capacity M-Discs where you want -# each disc to be decryptable on its own. +# 1. Scans all files in a source directory. +# 2. Groups them into "chunks" so that each chunk is <= a specified size (default 100GB). +# 3. Creates a TAR archive of each chunk, compresses it with lz4, and encrypts it with GPG (AES256). +# 4. Each .tar.lz4.gpg is fully independent (no other parts/discs needed to restore that chunk). +# 5. (Optional) Creates ISO images from each encrypted chunk if --create-iso is provided. +# 6. (Optional) Burns each chunk or ISO to M-Disc if --burn is provided. # -# Requirements: +# Usage: +# ./backup2mdisc.sh /path/to/source /path/to/destination [CHUNK_SIZE] [--create-iso] [--burn] +# +# Examples: +# ./backup2mdisc.sh /home/user/data /mnt/backup 100G --create-iso +# ./backup2mdisc.sh /data /backup 50G --burn +# +# Dependencies: # - bash # - gpg (for encryption) # - lz4 (for fast compression) # - tar +# - split or file-based grouping approach # - sha256sum (or 'shasum -a 256' on macOS/FreeBSD) +# - genisoimage or mkisofs (for creating ISOs if --create-iso) +# - growisofs (Linux) or hdiutil (macOS) for burning if --burn # -# Usage: -# ./backup2mdisc.sh /path/to/source /path/to/destination [chunk_size] -# -# Example: -# ./backup2mdisc.sh /home/user/documents /mnt/backup 100G -# -# Tips: -# - If you want to burn these archives to disc afterward, you can: -# genisoimage -o chunk_001.iso chunk_001.tar.lz4.gpg -# Then burn the ISO using growisofs or hdiutil, etc. -# -# - Each chunk is standalone. If chunk #3 is lost, the rest are unaffected, -# but you lose only the files on chunk #3. -# -# - If you have a file larger than 'chunk_size', this script won't handle it -# gracefully. You'd need to adjust or handle large files differently. +# Notes: +# - This script sorts files by size and accumulates them until the chunk is "full." +# - If a file alone is bigger than CHUNK_SIZE, this script won't handle it gracefully. +# - Each chunk gets a separate .tar.lz4.gpg file. If one disc is lost, only that chunk's files are lost. +# - Keep your GPG passphrase safe; you'll need it to decrypt any chunk. # set -e -# Default chunk size -DEFAULT_CHUNK_SIZE="100G" - ##################################### -# HELPER FUNCTIONS # +# CONFIGURATION & DEFAULTS # ##################################### +DEFAULT_CHUNK_SIZE="100G" # Adjust if you want a different default +MANIFEST_NAME="manifest_individual_chunks.txt" + +##################################### +# FUNCTIONS # +##################################### + +function usage() { + echo "Usage: $0 /path/to/source /path/to/destination [CHUNK_SIZE] [--create-iso] [--burn]" + echo + echo "Example: $0 /home/user/docs /mnt/backup 100G --create-iso --burn" + exit 1 +} + +# Cross-platform SHA-256 function compute_sha256() { if command -v sha256sum >/dev/null 2>&1; then sha256sum "$1" else - # macOS/FreeBSD fallback: shasum -a 256 "$1" fi } -function usage() { - echo "Usage: $0 /path/to/source /path/to/destination [chunk_size]" - echo "Example: $0 /data /backup 100G" - exit 1 -} - ##################################### -# MAIN PROGRAM # +# MAIN SCRIPT # ##################################### -# Parse arguments +# Parse primary arguments SOURCE_DIR="$1" DEST_DIR="$2" CHUNK_SIZE="${3:-$DEFAULT_CHUNK_SIZE}" +# Shift away the first 3 arguments if present +shift 3 || true + +CREATE_ISO=false +BURN_MEDIA=false + +# Parse flags +for arg in "$@"; do + case "$arg" in + --create-iso) + CREATE_ISO=true + ;; + --burn) + BURN_MEDIA=true + ;; + *) + ;; + esac +done + +# Basic checks if [[ -z "$SOURCE_DIR" || -z "$DEST_DIR" ]]; then usage fi if [[ ! -d "$SOURCE_DIR" ]]; then - echo "ERROR: Source directory does not exist: $SOURCE_DIR" + echo "ERROR: Source directory '$SOURCE_DIR' does not exist." exit 1 fi if [[ ! -d "$DEST_DIR" ]]; then - echo "ERROR: Destination directory does not exist: $DEST_DIR" + echo "ERROR: Destination directory '$DEST_DIR' does not exist." exit 1 fi -# Prompt for GPG passphrase (do not store in script) +# Prompt for GPG passphrase echo -n "Enter GPG passphrase (will not be displayed): " read -s GPG_PASSPHRASE echo -# Create a working subdir +# Create a working directory WORK_DIR="${DEST_DIR}/individual_chunks_$(date +%Y%m%d_%H%M%S)" mkdir -p "$WORK_DIR" -# This file will track which files are in which chunk, plus checksums -MANIFEST_FILE="${WORK_DIR}/manifest_individual_chunks.txt" +# Create a manifest file to track chunk -> files mapping and checksums +MANIFEST_FILE="${WORK_DIR}/${MANIFEST_NAME}" touch "$MANIFEST_FILE" -echo "Manifest for individual-chunk backup" > "$MANIFEST_FILE" +echo "Manifest for independent chunks backup" > "$MANIFEST_FILE" echo "Source: $SOURCE_DIR" >> "$MANIFEST_FILE" echo "Timestamp: $(date)" >> "$MANIFEST_FILE" -echo "Chunk size: $CHUNK_SIZE" >> "$MANIFEST_FILE" +echo "Chunk size limit: $CHUNK_SIZE" >> "$MANIFEST_FILE" echo >> "$MANIFEST_FILE" -# List of all files with size, sorted by file size ascending -# If you prefer alphabetical, remove the "-printf '%s %p\n'| sort -n" logic -FILE_LIST=$(mktemp) -find "$SOURCE_DIR" -type f -printf "%s %p\n" | sort -n > "$FILE_LIST" +# Step 1: Collect all files with their sizes and sort them (ascending by size). +TEMP_FILE_LIST=$(mktemp) +find "$SOURCE_DIR" -type f -printf "%s %p\n" | sort -n > "$TEMP_FILE_LIST" CHUNK_INDEX=1 CURRENT_CHUNK_SIZE=0 -TMP_FILELIST=$(mktemp) +TMP_CHUNK_LIST=$(mktemp) + +function bytes_from_iec() { + # Convert something like '100G' or '50G' into bytes using numfmt + numfmt --from=iec "$1" +} + +MAX_CHUNK_BYTES=$(bytes_from_iec "$CHUNK_SIZE") function start_new_chunk() { - # We'll reset the chunk accumulators - rm -f "$TMP_FILELIST" - touch "$TMP_FILELIST" + rm -f "$TMP_CHUNK_LIST" + touch "$TMP_CHUNK_LIST" CURRENT_CHUNK_SIZE=0 } -# Initialize chunk +function finalize_chunk() { + # Called when we have a list of files in TMP_CHUNK_LIST and we want to + # 1) TAR them + # 2) Compress with lz4 + # 3) Encrypt with GPG + # 4) Possibly create ISO + # 5) Possibly burn + # 6) Update manifest + + local chunk_name + chunk_name=$(printf "chunk_%03d.tar.lz4.gpg" "$CHUNK_INDEX") + + echo + echo "==> Creating chunk #$CHUNK_INDEX: $chunk_name" + + # Tar + lz4 + gpg pipeline + tar -cf - -T "$TMP_CHUNK_LIST" \ + | lz4 -c \ + | gpg --batch --yes --cipher-algo AES256 --passphrase "$GPG_PASSPHRASE" -c \ + > "${WORK_DIR}/${chunk_name}" + + # Generate a SHA-256 sum + local chunk_path="${WORK_DIR}/${chunk_name}" + local sum_line + sum_line=$(compute_sha256 "$chunk_path") + + # Add chunk info to manifest + echo "Chunk #$CHUNK_INDEX -> $chunk_name" >> "$MANIFEST_FILE" + echo "Files in this chunk:" >> "$MANIFEST_FILE" + cat "$TMP_CHUNK_LIST" >> "$MANIFEST_FILE" + echo "" >> "$MANIFEST_FILE" + echo "SHA256: $sum_line" >> "$MANIFEST_FILE" + echo "-----------------------------------" >> "$MANIFEST_FILE" + echo >> "$MANIFEST_FILE" + + # Optionally create ISO + local iso_name + iso_name=$(printf "chunk_%03d.iso" "$CHUNK_INDEX") + if [ "$CREATE_ISO" = true ]; then + echo "==> Creating ISO for chunk #$CHUNK_INDEX" + mkdir -p "${WORK_DIR}/iso_chunks" + local temp_iso_dir="${WORK_DIR}/temp_iso_dir_$CHUNK_INDEX" + mkdir -p "$temp_iso_dir" + + # Copy the encrypted archive into a temp directory + cp "$chunk_path" "$temp_iso_dir"/ + + # Build the ISO + local iso_output="${WORK_DIR}/iso_chunks/${iso_name}" + if command -v genisoimage >/dev/null 2>&1; then + genisoimage -quiet -o "$iso_output" -V "ENCRYPTED_BACKUP_${CHUNK_INDEX}" "$temp_iso_dir" + else + # Try mkisofs + mkisofs -quiet -o "$iso_output" -V "ENCRYPTED_BACKUP_${CHUNK_INDEX}" "$temp_iso_dir" + fi + rm -rf "$temp_iso_dir" + + # If --burn is also requested, burn the ISO + if [ "$BURN_MEDIA" = true ]; then + echo + echo "Please insert a blank M-Disc for chunk #$CHUNK_INDEX (ISO): $iso_name" + read -rp "Press [Enter] when ready to burn..." + if command -v growisofs >/dev/null 2>&1; then + growisofs -Z /dev/sr0="$iso_output" + elif [[ "$OSTYPE" == "darwin"* ]]; then + # macOS example + hdiutil burn "$iso_output" + else + echo "No recognized burner found. Please burn ${iso_output} manually." + fi + fi + else + # If we are not creating ISO but we are burning the chunk file directly + if [ "$BURN_MEDIA" = true ]; then + echo + echo "Please insert a blank M-Disc for chunk #$CHUNK_INDEX: $chunk_name" + read -rp "Press [Enter] when ready to burn..." + if command -v growisofs >/dev/null 2>&1; then + growisofs -Z /dev/sr0="$chunk_path" + elif [[ "$OSTYPE" == "darwin"* ]]; then + # hdiutil doesn't burn a raw file easily, typically it expects .iso + echo "On macOS, consider creating an ISO or using a different burning tool for $chunk_name." + else + echo "No recognized burner found. Please burn ${chunk_path} manually." + fi + fi + fi + + ((CHUNK_INDEX++)) + start_new_chunk +} + +# Initialize the first chunk start_new_chunk -while read -r line; do +# Step 2: Go through each file, add to chunk if it fits, otherwise finalize and start a new chunk. +while IFS= read -r line; do FILE_SIZE=$(echo "$line" | awk '{print $1}') FILE_PATH=$(echo "$line" | cut -d' ' -f2-) - # If adding this file exceeds chunk size, finalize the current chunk first - if [[ $(( CURRENT_CHUNK_SIZE + FILE_SIZE )) -gt $(( $(numfmt --from=iec $CHUNK_SIZE) )) ]]; then - # Finalize the chunk - # 1) Tar all the files in TMP_FILELIST - # 2) Compress with lz4 - # 3) Encrypt with gpg - # 4) Output a .tar.lz4.gpg in WORK_DIR - CHUNK_NAME=$(printf "chunk_%03d.tar.lz4.gpg" $CHUNK_INDEX) - - echo "==> Creating chunk #$CHUNK_INDEX with the collected files..." - tar -cf - -T "$TMP_FILELIST" \ - | lz4 -c \ - | gpg --batch --yes --cipher-algo AES256 --passphrase "$GPG_PASSPHRASE" -c \ - > "${WORK_DIR}/${CHUNK_NAME}" - - # Compute checksum & record - CHUNK_SHA256=$(compute_sha256 "${WORK_DIR}/${CHUNK_NAME}") - echo "Chunk #$CHUNK_INDEX -> ${CHUNK_NAME}" >> "$MANIFEST_FILE" - echo "$CHUNK_SHA256" >> "$MANIFEST_FILE" - echo >> "$MANIFEST_FILE" - - ((CHUNK_INDEX++)) - start_new_chunk + # If adding this file exceeds the chunk limit, finalize the current chunk now + if [[ $((CURRENT_CHUNK_SIZE + FILE_SIZE)) -gt $MAX_CHUNK_BYTES ]]; then + # Finalize current chunk if it has at least 1 file + if [[ $(wc -l < "$TMP_CHUNK_LIST") -gt 0 ]]; then + finalize_chunk + fi fi - # Add current file to the chunk - echo "$FILE_PATH" >> "$TMP_FILELIST" - CURRENT_CHUNK_SIZE=$(( CURRENT_CHUNK_SIZE + FILE_SIZE )) -done < "$FILE_LIST" + # Add the file to the chunk + echo "$FILE_PATH" >> "$TMP_CHUNK_LIST" + CURRENT_CHUNK_SIZE=$((CURRENT_CHUNK_SIZE + FILE_SIZE)) +done < "$TEMP_FILE_LIST" -# If TMP_FILELIST still has leftover files, finalize the last chunk -LAST_LIST_SIZE=$(wc -l < "$TMP_FILELIST") -if [[ "$LAST_LIST_SIZE" -gt 0 ]]; then - CHUNK_NAME=$(printf "chunk_%03d.tar.lz4.gpg" $CHUNK_INDEX) - echo "==> Creating final chunk #$CHUNK_INDEX..." - tar -cf - -T "$TMP_FILELIST" \ - | lz4 -c \ - | gpg --batch --yes --cipher-algo AES256 --passphrase "$GPG_PASSPHRASE" -c \ - > "${WORK_DIR}/${CHUNK_NAME}" - - # Compute checksum & record - CHUNK_SHA256=$(compute_sha256 "${WORK_DIR}/${CHUNK_NAME}") - echo "Chunk #$CHUNK_INDEX -> ${CHUNK_NAME}" >> "$MANIFEST_FILE" - echo "$CHUNK_SHA256" >> "$MANIFEST_FILE" - echo >> "$MANIFEST_FILE" +# Finalize the last chunk if it has leftover files +if [[ $(wc -l < "$TMP_CHUNK_LIST") -gt 0 ]]; then + finalize_chunk fi -echo "=== All Chunks Created ===" -echo "Chunks and manifest are located in: $WORK_DIR" -echo "Manifest file: $MANIFEST_FILE" +echo +echo "=== All chunks created ===" +echo "Your chunks (and possibly ISOs) are located in:" +echo " $WORK_DIR" +echo +echo "Manifest: $MANIFEST_FILE" +echo "-----------------------------------" +echo "Done!" # Cleanup -rm -f "$FILE_LIST" "$TMP_FILELIST" +rm -f "$TEMP_FILE_LIST" "$TMP_CHUNK_LIST" exit 0 \ No newline at end of file