re-add burn and create-iso
Signed-off-by: first <first@noreply.git.r21.io>
This commit is contained in:
parent
df214ea14c
commit
7ff6fd9a75
296
backup2mdisc.sh
296
backup2mdisc.sh
|
|
@ -3,175 +3,277 @@
|
|||
# backup2mdisc.sh
|
||||
#
|
||||
# Purpose:
|
||||
# Creates multiple self-contained 100GB (default) backup archives, each encrypted
|
||||
# independently. Useful for writing to large-capacity M-Discs where you want
|
||||
# each disc to be decryptable on its own.
|
||||
# 1. Scans all files in a source directory.
|
||||
# 2. Groups them into "chunks" so that each chunk is <= a specified size (default 100GB).
|
||||
# 3. Creates a TAR archive of each chunk, compresses it with lz4, and encrypts it with GPG (AES256).
|
||||
# 4. Each .tar.lz4.gpg is fully independent (no other parts/discs needed to restore that chunk).
|
||||
# 5. (Optional) Creates ISO images from each encrypted chunk if --create-iso is provided.
|
||||
# 6. (Optional) Burns each chunk or ISO to M-Disc if --burn is provided.
|
||||
#
|
||||
# Requirements:
|
||||
# Usage:
|
||||
# ./backup2mdisc.sh /path/to/source /path/to/destination [CHUNK_SIZE] [--create-iso] [--burn]
|
||||
#
|
||||
# Examples:
|
||||
# ./backup2mdisc.sh /home/user/data /mnt/backup 100G --create-iso
|
||||
# ./backup2mdisc.sh /data /backup 50G --burn
|
||||
#
|
||||
# Dependencies:
|
||||
# - bash
|
||||
# - gpg (for encryption)
|
||||
# - lz4 (for fast compression)
|
||||
# - tar
|
||||
# - split or file-based grouping approach
|
||||
# - sha256sum (or 'shasum -a 256' on macOS/FreeBSD)
|
||||
# - genisoimage or mkisofs (for creating ISOs if --create-iso)
|
||||
# - growisofs (Linux) or hdiutil (macOS) for burning if --burn
|
||||
#
|
||||
# Usage:
|
||||
# ./backup2mdisc.sh /path/to/source /path/to/destination [chunk_size]
|
||||
#
|
||||
# Example:
|
||||
# ./backup2mdisc.sh /home/user/documents /mnt/backup 100G
|
||||
#
|
||||
# Tips:
|
||||
# - If you want to burn these archives to disc afterward, you can:
|
||||
# genisoimage -o chunk_001.iso chunk_001.tar.lz4.gpg
|
||||
# Then burn the ISO using growisofs or hdiutil, etc.
|
||||
#
|
||||
# - Each chunk is standalone. If chunk #3 is lost, the rest are unaffected,
|
||||
# but you lose only the files on chunk #3.
|
||||
#
|
||||
# - If you have a file larger than 'chunk_size', this script won't handle it
|
||||
# gracefully. You'd need to adjust or handle large files differently.
|
||||
# Notes:
|
||||
# - This script sorts files by size and accumulates them until the chunk is "full."
|
||||
# - If a file alone is bigger than CHUNK_SIZE, this script won't handle it gracefully.
|
||||
# - Each chunk gets a separate .tar.lz4.gpg file. If one disc is lost, only that chunk's files are lost.
|
||||
# - Keep your GPG passphrase safe; you'll need it to decrypt any chunk.
|
||||
#
|
||||
|
||||
set -e
|
||||
|
||||
# Default chunk size
|
||||
DEFAULT_CHUNK_SIZE="100G"
|
||||
|
||||
#####################################
|
||||
# HELPER FUNCTIONS #
|
||||
# CONFIGURATION & DEFAULTS #
|
||||
#####################################
|
||||
|
||||
DEFAULT_CHUNK_SIZE="100G" # Adjust if you want a different default
|
||||
MANIFEST_NAME="manifest_individual_chunks.txt"
|
||||
|
||||
#####################################
|
||||
# FUNCTIONS #
|
||||
#####################################
|
||||
|
||||
function usage() {
|
||||
echo "Usage: $0 /path/to/source /path/to/destination [CHUNK_SIZE] [--create-iso] [--burn]"
|
||||
echo
|
||||
echo "Example: $0 /home/user/docs /mnt/backup 100G --create-iso --burn"
|
||||
exit 1
|
||||
}
|
||||
|
||||
# Cross-platform SHA-256
|
||||
function compute_sha256() {
|
||||
if command -v sha256sum >/dev/null 2>&1; then
|
||||
sha256sum "$1"
|
||||
else
|
||||
# macOS/FreeBSD fallback:
|
||||
shasum -a 256 "$1"
|
||||
fi
|
||||
}
|
||||
|
||||
function usage() {
|
||||
echo "Usage: $0 /path/to/source /path/to/destination [chunk_size]"
|
||||
echo "Example: $0 /data /backup 100G"
|
||||
exit 1
|
||||
}
|
||||
|
||||
#####################################
|
||||
# MAIN PROGRAM #
|
||||
# MAIN SCRIPT #
|
||||
#####################################
|
||||
|
||||
# Parse arguments
|
||||
# Parse primary arguments
|
||||
SOURCE_DIR="$1"
|
||||
DEST_DIR="$2"
|
||||
CHUNK_SIZE="${3:-$DEFAULT_CHUNK_SIZE}"
|
||||
|
||||
# Shift away the first 3 arguments if present
|
||||
shift 3 || true
|
||||
|
||||
CREATE_ISO=false
|
||||
BURN_MEDIA=false
|
||||
|
||||
# Parse flags
|
||||
for arg in "$@"; do
|
||||
case "$arg" in
|
||||
--create-iso)
|
||||
CREATE_ISO=true
|
||||
;;
|
||||
--burn)
|
||||
BURN_MEDIA=true
|
||||
;;
|
||||
*)
|
||||
;;
|
||||
esac
|
||||
done
|
||||
|
||||
# Basic checks
|
||||
if [[ -z "$SOURCE_DIR" || -z "$DEST_DIR" ]]; then
|
||||
usage
|
||||
fi
|
||||
|
||||
if [[ ! -d "$SOURCE_DIR" ]]; then
|
||||
echo "ERROR: Source directory does not exist: $SOURCE_DIR"
|
||||
echo "ERROR: Source directory '$SOURCE_DIR' does not exist."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [[ ! -d "$DEST_DIR" ]]; then
|
||||
echo "ERROR: Destination directory does not exist: $DEST_DIR"
|
||||
echo "ERROR: Destination directory '$DEST_DIR' does not exist."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Prompt for GPG passphrase (do not store in script)
|
||||
# Prompt for GPG passphrase
|
||||
echo -n "Enter GPG passphrase (will not be displayed): "
|
||||
read -s GPG_PASSPHRASE
|
||||
echo
|
||||
|
||||
# Create a working subdir
|
||||
# Create a working directory
|
||||
WORK_DIR="${DEST_DIR}/individual_chunks_$(date +%Y%m%d_%H%M%S)"
|
||||
mkdir -p "$WORK_DIR"
|
||||
|
||||
# This file will track which files are in which chunk, plus checksums
|
||||
MANIFEST_FILE="${WORK_DIR}/manifest_individual_chunks.txt"
|
||||
# Create a manifest file to track chunk -> files mapping and checksums
|
||||
MANIFEST_FILE="${WORK_DIR}/${MANIFEST_NAME}"
|
||||
touch "$MANIFEST_FILE"
|
||||
echo "Manifest for individual-chunk backup" > "$MANIFEST_FILE"
|
||||
echo "Manifest for independent chunks backup" > "$MANIFEST_FILE"
|
||||
echo "Source: $SOURCE_DIR" >> "$MANIFEST_FILE"
|
||||
echo "Timestamp: $(date)" >> "$MANIFEST_FILE"
|
||||
echo "Chunk size: $CHUNK_SIZE" >> "$MANIFEST_FILE"
|
||||
echo "Chunk size limit: $CHUNK_SIZE" >> "$MANIFEST_FILE"
|
||||
echo >> "$MANIFEST_FILE"
|
||||
|
||||
# List of all files with size, sorted by file size ascending
|
||||
# If you prefer alphabetical, remove the "-printf '%s %p\n'| sort -n" logic
|
||||
FILE_LIST=$(mktemp)
|
||||
find "$SOURCE_DIR" -type f -printf "%s %p\n" | sort -n > "$FILE_LIST"
|
||||
# Step 1: Collect all files with their sizes and sort them (ascending by size).
|
||||
TEMP_FILE_LIST=$(mktemp)
|
||||
find "$SOURCE_DIR" -type f -printf "%s %p\n" | sort -n > "$TEMP_FILE_LIST"
|
||||
|
||||
CHUNK_INDEX=1
|
||||
CURRENT_CHUNK_SIZE=0
|
||||
TMP_FILELIST=$(mktemp)
|
||||
TMP_CHUNK_LIST=$(mktemp)
|
||||
|
||||
function bytes_from_iec() {
|
||||
# Convert something like '100G' or '50G' into bytes using numfmt
|
||||
numfmt --from=iec "$1"
|
||||
}
|
||||
|
||||
MAX_CHUNK_BYTES=$(bytes_from_iec "$CHUNK_SIZE")
|
||||
|
||||
function start_new_chunk() {
|
||||
# We'll reset the chunk accumulators
|
||||
rm -f "$TMP_FILELIST"
|
||||
touch "$TMP_FILELIST"
|
||||
rm -f "$TMP_CHUNK_LIST"
|
||||
touch "$TMP_CHUNK_LIST"
|
||||
CURRENT_CHUNK_SIZE=0
|
||||
}
|
||||
|
||||
# Initialize chunk
|
||||
function finalize_chunk() {
|
||||
# Called when we have a list of files in TMP_CHUNK_LIST and we want to
|
||||
# 1) TAR them
|
||||
# 2) Compress with lz4
|
||||
# 3) Encrypt with GPG
|
||||
# 4) Possibly create ISO
|
||||
# 5) Possibly burn
|
||||
# 6) Update manifest
|
||||
|
||||
local chunk_name
|
||||
chunk_name=$(printf "chunk_%03d.tar.lz4.gpg" "$CHUNK_INDEX")
|
||||
|
||||
echo
|
||||
echo "==> Creating chunk #$CHUNK_INDEX: $chunk_name"
|
||||
|
||||
# Tar + lz4 + gpg pipeline
|
||||
tar -cf - -T "$TMP_CHUNK_LIST" \
|
||||
| lz4 -c \
|
||||
| gpg --batch --yes --cipher-algo AES256 --passphrase "$GPG_PASSPHRASE" -c \
|
||||
> "${WORK_DIR}/${chunk_name}"
|
||||
|
||||
# Generate a SHA-256 sum
|
||||
local chunk_path="${WORK_DIR}/${chunk_name}"
|
||||
local sum_line
|
||||
sum_line=$(compute_sha256 "$chunk_path")
|
||||
|
||||
# Add chunk info to manifest
|
||||
echo "Chunk #$CHUNK_INDEX -> $chunk_name" >> "$MANIFEST_FILE"
|
||||
echo "Files in this chunk:" >> "$MANIFEST_FILE"
|
||||
cat "$TMP_CHUNK_LIST" >> "$MANIFEST_FILE"
|
||||
echo "" >> "$MANIFEST_FILE"
|
||||
echo "SHA256: $sum_line" >> "$MANIFEST_FILE"
|
||||
echo "-----------------------------------" >> "$MANIFEST_FILE"
|
||||
echo >> "$MANIFEST_FILE"
|
||||
|
||||
# Optionally create ISO
|
||||
local iso_name
|
||||
iso_name=$(printf "chunk_%03d.iso" "$CHUNK_INDEX")
|
||||
if [ "$CREATE_ISO" = true ]; then
|
||||
echo "==> Creating ISO for chunk #$CHUNK_INDEX"
|
||||
mkdir -p "${WORK_DIR}/iso_chunks"
|
||||
local temp_iso_dir="${WORK_DIR}/temp_iso_dir_$CHUNK_INDEX"
|
||||
mkdir -p "$temp_iso_dir"
|
||||
|
||||
# Copy the encrypted archive into a temp directory
|
||||
cp "$chunk_path" "$temp_iso_dir"/
|
||||
|
||||
# Build the ISO
|
||||
local iso_output="${WORK_DIR}/iso_chunks/${iso_name}"
|
||||
if command -v genisoimage >/dev/null 2>&1; then
|
||||
genisoimage -quiet -o "$iso_output" -V "ENCRYPTED_BACKUP_${CHUNK_INDEX}" "$temp_iso_dir"
|
||||
else
|
||||
# Try mkisofs
|
||||
mkisofs -quiet -o "$iso_output" -V "ENCRYPTED_BACKUP_${CHUNK_INDEX}" "$temp_iso_dir"
|
||||
fi
|
||||
rm -rf "$temp_iso_dir"
|
||||
|
||||
# If --burn is also requested, burn the ISO
|
||||
if [ "$BURN_MEDIA" = true ]; then
|
||||
echo
|
||||
echo "Please insert a blank M-Disc for chunk #$CHUNK_INDEX (ISO): $iso_name"
|
||||
read -rp "Press [Enter] when ready to burn..."
|
||||
if command -v growisofs >/dev/null 2>&1; then
|
||||
growisofs -Z /dev/sr0="$iso_output"
|
||||
elif [[ "$OSTYPE" == "darwin"* ]]; then
|
||||
# macOS example
|
||||
hdiutil burn "$iso_output"
|
||||
else
|
||||
echo "No recognized burner found. Please burn ${iso_output} manually."
|
||||
fi
|
||||
fi
|
||||
else
|
||||
# If we are not creating ISO but we are burning the chunk file directly
|
||||
if [ "$BURN_MEDIA" = true ]; then
|
||||
echo
|
||||
echo "Please insert a blank M-Disc for chunk #$CHUNK_INDEX: $chunk_name"
|
||||
read -rp "Press [Enter] when ready to burn..."
|
||||
if command -v growisofs >/dev/null 2>&1; then
|
||||
growisofs -Z /dev/sr0="$chunk_path"
|
||||
elif [[ "$OSTYPE" == "darwin"* ]]; then
|
||||
# hdiutil doesn't burn a raw file easily, typically it expects .iso
|
||||
echo "On macOS, consider creating an ISO or using a different burning tool for $chunk_name."
|
||||
else
|
||||
echo "No recognized burner found. Please burn ${chunk_path} manually."
|
||||
fi
|
||||
fi
|
||||
fi
|
||||
|
||||
((CHUNK_INDEX++))
|
||||
start_new_chunk
|
||||
}
|
||||
|
||||
# Initialize the first chunk
|
||||
start_new_chunk
|
||||
|
||||
while read -r line; do
|
||||
# Step 2: Go through each file, add to chunk if it fits, otherwise finalize and start a new chunk.
|
||||
while IFS= read -r line; do
|
||||
FILE_SIZE=$(echo "$line" | awk '{print $1}')
|
||||
FILE_PATH=$(echo "$line" | cut -d' ' -f2-)
|
||||
|
||||
# If adding this file exceeds chunk size, finalize the current chunk first
|
||||
if [[ $(( CURRENT_CHUNK_SIZE + FILE_SIZE )) -gt $(( $(numfmt --from=iec $CHUNK_SIZE) )) ]]; then
|
||||
# Finalize the chunk
|
||||
# 1) Tar all the files in TMP_FILELIST
|
||||
# 2) Compress with lz4
|
||||
# 3) Encrypt with gpg
|
||||
# 4) Output a .tar.lz4.gpg in WORK_DIR
|
||||
CHUNK_NAME=$(printf "chunk_%03d.tar.lz4.gpg" $CHUNK_INDEX)
|
||||
|
||||
echo "==> Creating chunk #$CHUNK_INDEX with the collected files..."
|
||||
tar -cf - -T "$TMP_FILELIST" \
|
||||
| lz4 -c \
|
||||
| gpg --batch --yes --cipher-algo AES256 --passphrase "$GPG_PASSPHRASE" -c \
|
||||
> "${WORK_DIR}/${CHUNK_NAME}"
|
||||
|
||||
# Compute checksum & record
|
||||
CHUNK_SHA256=$(compute_sha256 "${WORK_DIR}/${CHUNK_NAME}")
|
||||
echo "Chunk #$CHUNK_INDEX -> ${CHUNK_NAME}" >> "$MANIFEST_FILE"
|
||||
echo "$CHUNK_SHA256" >> "$MANIFEST_FILE"
|
||||
echo >> "$MANIFEST_FILE"
|
||||
|
||||
((CHUNK_INDEX++))
|
||||
start_new_chunk
|
||||
# If adding this file exceeds the chunk limit, finalize the current chunk now
|
||||
if [[ $((CURRENT_CHUNK_SIZE + FILE_SIZE)) -gt $MAX_CHUNK_BYTES ]]; then
|
||||
# Finalize current chunk if it has at least 1 file
|
||||
if [[ $(wc -l < "$TMP_CHUNK_LIST") -gt 0 ]]; then
|
||||
finalize_chunk
|
||||
fi
|
||||
fi
|
||||
|
||||
# Add current file to the chunk
|
||||
echo "$FILE_PATH" >> "$TMP_FILELIST"
|
||||
CURRENT_CHUNK_SIZE=$(( CURRENT_CHUNK_SIZE + FILE_SIZE ))
|
||||
done < "$FILE_LIST"
|
||||
# Add the file to the chunk
|
||||
echo "$FILE_PATH" >> "$TMP_CHUNK_LIST"
|
||||
CURRENT_CHUNK_SIZE=$((CURRENT_CHUNK_SIZE + FILE_SIZE))
|
||||
done < "$TEMP_FILE_LIST"
|
||||
|
||||
# If TMP_FILELIST still has leftover files, finalize the last chunk
|
||||
LAST_LIST_SIZE=$(wc -l < "$TMP_FILELIST")
|
||||
if [[ "$LAST_LIST_SIZE" -gt 0 ]]; then
|
||||
CHUNK_NAME=$(printf "chunk_%03d.tar.lz4.gpg" $CHUNK_INDEX)
|
||||
echo "==> Creating final chunk #$CHUNK_INDEX..."
|
||||
tar -cf - -T "$TMP_FILELIST" \
|
||||
| lz4 -c \
|
||||
| gpg --batch --yes --cipher-algo AES256 --passphrase "$GPG_PASSPHRASE" -c \
|
||||
> "${WORK_DIR}/${CHUNK_NAME}"
|
||||
|
||||
# Compute checksum & record
|
||||
CHUNK_SHA256=$(compute_sha256 "${WORK_DIR}/${CHUNK_NAME}")
|
||||
echo "Chunk #$CHUNK_INDEX -> ${CHUNK_NAME}" >> "$MANIFEST_FILE"
|
||||
echo "$CHUNK_SHA256" >> "$MANIFEST_FILE"
|
||||
echo >> "$MANIFEST_FILE"
|
||||
# Finalize the last chunk if it has leftover files
|
||||
if [[ $(wc -l < "$TMP_CHUNK_LIST") -gt 0 ]]; then
|
||||
finalize_chunk
|
||||
fi
|
||||
|
||||
echo "=== All Chunks Created ==="
|
||||
echo "Chunks and manifest are located in: $WORK_DIR"
|
||||
echo "Manifest file: $MANIFEST_FILE"
|
||||
echo
|
||||
echo "=== All chunks created ==="
|
||||
echo "Your chunks (and possibly ISOs) are located in:"
|
||||
echo " $WORK_DIR"
|
||||
echo
|
||||
echo "Manifest: $MANIFEST_FILE"
|
||||
echo "-----------------------------------"
|
||||
echo "Done!"
|
||||
|
||||
# Cleanup
|
||||
rm -f "$FILE_LIST" "$TMP_FILELIST"
|
||||
rm -f "$TEMP_FILE_LIST" "$TMP_CHUNK_LIST"
|
||||
|
||||
exit 0
|
||||
Loading…
Reference in a new issue