From e0d17779655b62b7f284379dffcdbe53610fb55e Mon Sep 17 00:00:00 2001 From: first Date: Fri, 24 Jan 2025 06:46:04 +0000 Subject: [PATCH] Update backup2mdisc.sh Signed-off-by: first --- backup2mdisc.sh | 288 +++++++++++++++++++----------------------------- 1 file changed, 111 insertions(+), 177 deletions(-) diff --git a/backup2mdisc.sh b/backup2mdisc.sh index 5099aa1..bc41de8 100644 --- a/backup2mdisc.sh +++ b/backup2mdisc.sh @@ -1,243 +1,177 @@ #!/usr/bin/env bash # -# backup_to_mdisc.sh +# backup2mdisc.sh # -# A script to: -# 1. Archive a directory. -# 2. Compress and encrypt it. -# 3. Split into chunks (default 100GB). -# 4. Generate checksums and a manifest. -# 5. Optionally create ISO images for burning to M-Disc. +# Purpose: +# Creates multiple self-contained 100GB (default) backup archives, each encrypted +# independently. Useful for writing to large-capacity M-Discs where you want +# each disc to be decryptable on its own. +# +# Requirements: +# - bash +# - gpg (for encryption) +# - lz4 (for fast compression) +# - tar +# - sha256sum (or 'shasum -a 256' on macOS/FreeBSD) # # Usage: -# ./backup_to_mdisc.sh /path/to/source /path/to/destination [CHUNK_SIZE] [--create-iso] [--burn] +# ./backup2mdisc.sh /path/to/source /path/to/destination [chunk_size] # -# Examples: -# ./backup_to_mdisc.sh /home/user/documents /media/backup 100G --create-iso -# ./backup_to_mdisc.sh /data /backup 100G --create-iso --burn +# Example: +# ./backup2mdisc.sh /home/user/documents /mnt/backup 100G # -# Dependencies (install these if missing): -# - tar -# - xz -# - gpg -# - split -# - sha256sum (on BSD/macOS, use 'shasum -a 256') -# - genisoimage or mkisofs (for ISO creation) -# - growisofs (on Linux, for burning) -# - hdiutil (on macOS, for burning) +# Tips: +# - If you want to burn these archives to disc afterward, you can: +# genisoimage -o chunk_001.iso chunk_001.tar.lz4.gpg +# Then burn the ISO using growisofs or hdiutil, etc. +# +# - Each chunk is standalone. If chunk #3 is lost, the rest are unaffected, +# but you lose only the files on chunk #3. +# +# - If you have a file larger than 'chunk_size', this script won't handle it +# gracefully. You'd need to adjust or handle large files differently. # -# NOTE: -# - This script assumes you have a functioning optical burner that supports -# 100GB M-Disc (BD-XL) media, and the relevant drivers and software installed. -# - For FreeBSD, adjust commands (e.g., use 'shasum -a 256' instead of 'sha256sum'). set -e -##################################### -# CONFIGURATION # -##################################### - -# The default chunk size for splitting. 100GB is typical for 100GB M-Disc (BD-XL). +# Default chunk size DEFAULT_CHUNK_SIZE="100G" -# The name for the final TAR.XZ.GPG file (before splitting). -# This will be suffixed by .partNN once split. -ENCRYPTED_ARCHIVE_NAME="backup.tar.xz.gpg" - -# The manifest file name. -MANIFEST_NAME="backup_manifest.txt" - ##################################### # HELPER FUNCTIONS # ##################################### -# Cross-platform SHA-256 function. On Linux, we can rely on 'sha256sum'. -# On macOS/FreeBSD, you might need to use 'shasum -a 256'. function compute_sha256() { if command -v sha256sum >/dev/null 2>&1; then sha256sum "$1" else + # macOS/FreeBSD fallback: shasum -a 256 "$1" fi } -##################################### -# MAIN SCRIPT # -##################################### - function usage() { - echo "Usage: $0 /path/to/source /path/to/destination [CHUNK_SIZE] [--create-iso] [--burn]" - echo - echo "Example: $0 /home/user/docs /mnt/backup 100G --create-iso --burn" + echo "Usage: $0 /path/to/source /path/to/destination [chunk_size]" + echo "Example: $0 /data /backup 100G" exit 1 } +##################################### +# MAIN PROGRAM # +##################################### + # Parse arguments SOURCE_DIR="$1" DEST_DIR="$2" CHUNK_SIZE="${3:-$DEFAULT_CHUNK_SIZE}" -shift 3 || true - -CREATE_ISO=false -BURN_MEDIA=false - -for arg in "$@"; do - case "$arg" in - --create-iso) - CREATE_ISO=true - ;; - --burn) - BURN_MEDIA=true - ;; - *) - ;; - esac -done - -# Basic checks if [[ -z "$SOURCE_DIR" || -z "$DEST_DIR" ]]; then usage fi if [[ ! -d "$SOURCE_DIR" ]]; then - echo "ERROR: Source directory '$SOURCE_DIR' does not exist." + echo "ERROR: Source directory does not exist: $SOURCE_DIR" exit 1 fi if [[ ! -d "$DEST_DIR" ]]; then - echo "ERROR: Destination directory '$DEST_DIR' does not exist." + echo "ERROR: Destination directory does not exist: $DEST_DIR" exit 1 fi -# Prompt for GPG passphrase (don't store in script for security). +# Prompt for GPG passphrase (do not store in script) echo -n "Enter GPG passphrase (will not be displayed): " read -s GPG_PASSPHRASE echo -# Create a working directory inside DEST_DIR for the backup process -WORK_DIR="${DEST_DIR}/backup_$(date +%Y%m%d_%H%M%S)" +# Create a working subdir +WORK_DIR="${DEST_DIR}/individual_chunks_$(date +%Y%m%d_%H%M%S)" mkdir -p "$WORK_DIR" -cd "$WORK_DIR" -echo "=== Step 1: Create a compressed TAR archive and encrypt it with GPG ===" -echo "Creating encrypted archive. This may take a while depending on your data size..." +# This file will track which files are in which chunk, plus checksums +MANIFEST_FILE="${WORK_DIR}/manifest_individual_chunks.txt" +touch "$MANIFEST_FILE" +echo "Manifest for individual-chunk backup" > "$MANIFEST_FILE" +echo "Source: $SOURCE_DIR" >> "$MANIFEST_FILE" +echo "Timestamp: $(date)" >> "$MANIFEST_FILE" +echo "Chunk size: $CHUNK_SIZE" >> "$MANIFEST_FILE" +echo >> "$MANIFEST_FILE" -# We tar, compress, and encrypt in a single pipeline. -# tar -cf - : stream archive -# xz -c -9 : compress with xz at high compression -# gpg -c : symmetric encrypt, using passphrase -# -# Adjust cipher-algo or compression level (-9) as needed. +# List of all files with size, sorted by file size ascending +# If you prefer alphabetical, remove the "-printf '%s %p\n'| sort -n" logic +FILE_LIST=$(mktemp) +find "$SOURCE_DIR" -type f -printf "%s %p\n" | sort -n > "$FILE_LIST" -tar -cf - "$SOURCE_DIR" \ - | xz -c -9 \ - | gpg --batch --yes --cipher-algo AES256 --passphrase "$GPG_PASSPHRASE" -c \ - > "${ENCRYPTED_ARCHIVE_NAME}" +CHUNK_INDEX=1 +CURRENT_CHUNK_SIZE=0 +TMP_FILELIST=$(mktemp) -echo "=== Step 2: Split the encrypted archive into $CHUNK_SIZE chunks ===" -split -b "$CHUNK_SIZE" -a 3 "${ENCRYPTED_ARCHIVE_NAME}" "${ENCRYPTED_ARCHIVE_NAME}." +function start_new_chunk() { + # We'll reset the chunk accumulators + rm -f "$TMP_FILELIST" + touch "$TMP_FILELIST" + CURRENT_CHUNK_SIZE=0 +} -# Remove the single large file to save space (optional). -rm -f "${ENCRYPTED_ARCHIVE_NAME}" +# Initialize chunk +start_new_chunk -echo "=== Step 3: Generate checksums and a manifest/catalog ===" -touch "${MANIFEST_NAME}" +while read -r line; do + FILE_SIZE=$(echo "$line" | awk '{print $1}') + FILE_PATH=$(echo "$line" | cut -d' ' -f2-) -echo "Backup Manifest - $(date)" >> "${MANIFEST_NAME}" -echo "Source directory: $SOURCE_DIR" >> "${MANIFEST_NAME}" -echo "Destination directory: $DEST_DIR" >> "${MANIFEST_NAME}" -echo "Split chunk size: $CHUNK_SIZE" >> "${MANIFEST_NAME}" -echo "Encrypted archive chunk names:" >> "${MANIFEST_NAME}" -echo >> "${MANIFEST_NAME}" + # If adding this file exceeds chunk size, finalize the current chunk first + if [[ $(( CURRENT_CHUNK_SIZE + FILE_SIZE )) -gt $(( $(numfmt --from=iec $CHUNK_SIZE) )) ]]; then + # Finalize the chunk + # 1) Tar all the files in TMP_FILELIST + # 2) Compress with lz4 + # 3) Encrypt with gpg + # 4) Output a .tar.lz4.gpg in WORK_DIR + CHUNK_NAME=$(printf "chunk_%03d.tar.lz4.gpg" $CHUNK_INDEX) -for chunk in ${ENCRYPTED_ARCHIVE_NAME}.*; do - CHUNK_SHA256=$(compute_sha256 "$chunk") - echo "$CHUNK_SHA256" >> "${MANIFEST_NAME}" -done + echo "==> Creating chunk #$CHUNK_INDEX with the collected files..." + tar -cf - -T "$TMP_FILELIST" \ + | lz4 -c \ + | gpg --batch --yes --cipher-algo AES256 --passphrase "$GPG_PASSPHRASE" -c \ + > "${WORK_DIR}/${CHUNK_NAME}" -echo "Manifest created at: ${WORK_DIR}/${MANIFEST_NAME}" + # Compute checksum & record + CHUNK_SHA256=$(compute_sha256 "${WORK_DIR}/${CHUNK_NAME}") + echo "Chunk #$CHUNK_INDEX -> ${CHUNK_NAME}" >> "$MANIFEST_FILE" + echo "$CHUNK_SHA256" >> "$MANIFEST_FILE" + echo >> "$MANIFEST_FILE" -# If ISO creation is requested -if [ "$CREATE_ISO" = true ]; then - echo "=== Step 4: Create an ISO for each chunk (for easier burning) ===" - - # We'll place ISOs in a subfolder - mkdir -p iso_chunks - - for chunk in ${ENCRYPTED_ARCHIVE_NAME}.*; do - ISO_BASENAME="${chunk}.iso" - - # Create a temporary directory to hold the chunk file - mkdir -p temp_dir - cp "$chunk" temp_dir/ - - # Build an ISO with a single file inside: - genisoimage -o "iso_chunks/${ISO_BASENAME}" -V "ENCRYPTED_BACKUP" temp_dir >/dev/null 2>&1 || \ - mkisofs -o "iso_chunks/${ISO_BASENAME}" -V "ENCRYPTED_BACKUP" temp_dir - - # Remove the temporary directory - rm -rf temp_dir - done - - echo "ISO files created under: ${WORK_DIR}/iso_chunks" -fi - -# If burning is requested, attempt to burn right away. -# For cross-platform compatibility, we'll provide examples. -# You may need to adapt device names (/dev/sr0, /dev/dvd, etc.). - -if [ "$BURN_MEDIA" = true ]; then - echo "=== Step 5: Burn chunks/ISOs to M-Disc ===" - - # Example using growisofs on Linux: - # growisofs -Z /dev/sr0=chunk_or_iso - # or: - # growisofs -use-the-force-luke=dao -speed=2 -Z /dev/sr0=chunk_or_iso - - # Example using hdiutil on macOS for ISO: - # hdiutil burn chunk.iso - - echo "Attempting to burn each chunk (or ISO) to M-Disc. Please ensure a blank M-Disc is loaded each time." - - if [ "$CREATE_ISO" = true ]; then - # We have ISO images - for iso_file in iso_chunks/*.iso; do - echo "Insert new disc for: $iso_file" - read -p "Press [Enter] when ready to burn..." - - # Linux (growisofs) example: - if command -v growisofs >/dev/null 2>&1; then - growisofs -Z /dev/sr0="$iso_file" - elif [[ "$OSTYPE" == "darwin"* ]]; then - # macOS example using hdiutil - hdiutil burn "$iso_file" - else - echo "No known burner command found. Please burn manually: $iso_file" - fi - done - else - # Burn the chunk files directly. - for chunk in ${ENCRYPTED_ARCHIVE_NAME}.*; do - echo "Insert new disc for: $chunk" - read -p "Press [Enter] when ready to burn..." - - if command -v growisofs >/dev/null 2>&1; then - growisofs -Z /dev/sr0="$chunk" - elif [[ "$OSTYPE" == "darwin"* ]]; then - # We can't directly burn a raw file with hdiutil. Typically you'd create an ISO first. - # So warn the user. - echo "On macOS, please create an ISO or use a separate burning tool for: $chunk" - else - echo "No known burner command found. Please burn manually: $chunk" - fi - done + ((CHUNK_INDEX++)) + start_new_chunk fi - echo "Burning process completed. Verify your discs for peace of mind." + # Add current file to the chunk + echo "$FILE_PATH" >> "$TMP_FILELIST" + CURRENT_CHUNK_SIZE=$(( CURRENT_CHUNK_SIZE + FILE_SIZE )) +done < "$FILE_LIST" + +# If TMP_FILELIST still has leftover files, finalize the last chunk +LAST_LIST_SIZE=$(wc -l < "$TMP_FILELIST") +if [[ "$LAST_LIST_SIZE" -gt 0 ]]; then + CHUNK_NAME=$(printf "chunk_%03d.tar.lz4.gpg" $CHUNK_INDEX) + echo "==> Creating final chunk #$CHUNK_INDEX..." + tar -cf - -T "$TMP_FILELIST" \ + | lz4 -c \ + | gpg --batch --yes --cipher-algo AES256 --passphrase "$GPG_PASSPHRASE" -c \ + > "${WORK_DIR}/${CHUNK_NAME}" + + # Compute checksum & record + CHUNK_SHA256=$(compute_sha256 "${WORK_DIR}/${CHUNK_NAME}") + echo "Chunk #$CHUNK_INDEX -> ${CHUNK_NAME}" >> "$MANIFEST_FILE" + echo "$CHUNK_SHA256" >> "$MANIFEST_FILE" + echo >> "$MANIFEST_FILE" fi -echo "=== All done! ===" -echo "Your backup chunks and manifest are in: ${WORK_DIR}" -echo "Keep the manifest safe. You'll need all chunk files + passphrase to restore." +echo "=== All Chunks Created ===" +echo "Chunks and manifest are located in: $WORK_DIR" +echo "Manifest file: $MANIFEST_FILE" + +# Cleanup +rm -f "$FILE_LIST" "$TMP_FILELIST" + exit 0 \ No newline at end of file