From e0d17779655b62b7f284379dffcdbe53610fb55e Mon Sep 17 00:00:00 2001 From: first Date: Fri, 24 Jan 2025 06:46:04 +0000 Subject: [PATCH 1/5] Update backup2mdisc.sh Signed-off-by: first --- backup2mdisc.sh | 288 +++++++++++++++++++----------------------------- 1 file changed, 111 insertions(+), 177 deletions(-) diff --git a/backup2mdisc.sh b/backup2mdisc.sh index 5099aa1..bc41de8 100644 --- a/backup2mdisc.sh +++ b/backup2mdisc.sh @@ -1,243 +1,177 @@ #!/usr/bin/env bash # -# backup_to_mdisc.sh +# backup2mdisc.sh # -# A script to: -# 1. Archive a directory. -# 2. Compress and encrypt it. -# 3. Split into chunks (default 100GB). -# 4. Generate checksums and a manifest. -# 5. Optionally create ISO images for burning to M-Disc. +# Purpose: +# Creates multiple self-contained 100GB (default) backup archives, each encrypted +# independently. Useful for writing to large-capacity M-Discs where you want +# each disc to be decryptable on its own. +# +# Requirements: +# - bash +# - gpg (for encryption) +# - lz4 (for fast compression) +# - tar +# - sha256sum (or 'shasum -a 256' on macOS/FreeBSD) # # Usage: -# ./backup_to_mdisc.sh /path/to/source /path/to/destination [CHUNK_SIZE] [--create-iso] [--burn] +# ./backup2mdisc.sh /path/to/source /path/to/destination [chunk_size] # -# Examples: -# ./backup_to_mdisc.sh /home/user/documents /media/backup 100G --create-iso -# ./backup_to_mdisc.sh /data /backup 100G --create-iso --burn +# Example: +# ./backup2mdisc.sh /home/user/documents /mnt/backup 100G # -# Dependencies (install these if missing): -# - tar -# - xz -# - gpg -# - split -# - sha256sum (on BSD/macOS, use 'shasum -a 256') -# - genisoimage or mkisofs (for ISO creation) -# - growisofs (on Linux, for burning) -# - hdiutil (on macOS, for burning) +# Tips: +# - If you want to burn these archives to disc afterward, you can: +# genisoimage -o chunk_001.iso chunk_001.tar.lz4.gpg +# Then burn the ISO using growisofs or hdiutil, etc. +# +# - Each chunk is standalone. If chunk #3 is lost, the rest are unaffected, +# but you lose only the files on chunk #3. +# +# - If you have a file larger than 'chunk_size', this script won't handle it +# gracefully. You'd need to adjust or handle large files differently. # -# NOTE: -# - This script assumes you have a functioning optical burner that supports -# 100GB M-Disc (BD-XL) media, and the relevant drivers and software installed. -# - For FreeBSD, adjust commands (e.g., use 'shasum -a 256' instead of 'sha256sum'). set -e -##################################### -# CONFIGURATION # -##################################### - -# The default chunk size for splitting. 100GB is typical for 100GB M-Disc (BD-XL). +# Default chunk size DEFAULT_CHUNK_SIZE="100G" -# The name for the final TAR.XZ.GPG file (before splitting). -# This will be suffixed by .partNN once split. -ENCRYPTED_ARCHIVE_NAME="backup.tar.xz.gpg" - -# The manifest file name. -MANIFEST_NAME="backup_manifest.txt" - ##################################### # HELPER FUNCTIONS # ##################################### -# Cross-platform SHA-256 function. On Linux, we can rely on 'sha256sum'. -# On macOS/FreeBSD, you might need to use 'shasum -a 256'. function compute_sha256() { if command -v sha256sum >/dev/null 2>&1; then sha256sum "$1" else + # macOS/FreeBSD fallback: shasum -a 256 "$1" fi } -##################################### -# MAIN SCRIPT # -##################################### - function usage() { - echo "Usage: $0 /path/to/source /path/to/destination [CHUNK_SIZE] [--create-iso] [--burn]" - echo - echo "Example: $0 /home/user/docs /mnt/backup 100G --create-iso --burn" + echo "Usage: $0 /path/to/source /path/to/destination [chunk_size]" + echo "Example: $0 /data /backup 100G" exit 1 } +##################################### +# MAIN PROGRAM # +##################################### + # Parse arguments SOURCE_DIR="$1" DEST_DIR="$2" CHUNK_SIZE="${3:-$DEFAULT_CHUNK_SIZE}" -shift 3 || true - -CREATE_ISO=false -BURN_MEDIA=false - -for arg in "$@"; do - case "$arg" in - --create-iso) - CREATE_ISO=true - ;; - --burn) - BURN_MEDIA=true - ;; - *) - ;; - esac -done - -# Basic checks if [[ -z "$SOURCE_DIR" || -z "$DEST_DIR" ]]; then usage fi if [[ ! -d "$SOURCE_DIR" ]]; then - echo "ERROR: Source directory '$SOURCE_DIR' does not exist." + echo "ERROR: Source directory does not exist: $SOURCE_DIR" exit 1 fi if [[ ! -d "$DEST_DIR" ]]; then - echo "ERROR: Destination directory '$DEST_DIR' does not exist." + echo "ERROR: Destination directory does not exist: $DEST_DIR" exit 1 fi -# Prompt for GPG passphrase (don't store in script for security). +# Prompt for GPG passphrase (do not store in script) echo -n "Enter GPG passphrase (will not be displayed): " read -s GPG_PASSPHRASE echo -# Create a working directory inside DEST_DIR for the backup process -WORK_DIR="${DEST_DIR}/backup_$(date +%Y%m%d_%H%M%S)" +# Create a working subdir +WORK_DIR="${DEST_DIR}/individual_chunks_$(date +%Y%m%d_%H%M%S)" mkdir -p "$WORK_DIR" -cd "$WORK_DIR" -echo "=== Step 1: Create a compressed TAR archive and encrypt it with GPG ===" -echo "Creating encrypted archive. This may take a while depending on your data size..." +# This file will track which files are in which chunk, plus checksums +MANIFEST_FILE="${WORK_DIR}/manifest_individual_chunks.txt" +touch "$MANIFEST_FILE" +echo "Manifest for individual-chunk backup" > "$MANIFEST_FILE" +echo "Source: $SOURCE_DIR" >> "$MANIFEST_FILE" +echo "Timestamp: $(date)" >> "$MANIFEST_FILE" +echo "Chunk size: $CHUNK_SIZE" >> "$MANIFEST_FILE" +echo >> "$MANIFEST_FILE" -# We tar, compress, and encrypt in a single pipeline. -# tar -cf - : stream archive -# xz -c -9 : compress with xz at high compression -# gpg -c : symmetric encrypt, using passphrase -# -# Adjust cipher-algo or compression level (-9) as needed. +# List of all files with size, sorted by file size ascending +# If you prefer alphabetical, remove the "-printf '%s %p\n'| sort -n" logic +FILE_LIST=$(mktemp) +find "$SOURCE_DIR" -type f -printf "%s %p\n" | sort -n > "$FILE_LIST" -tar -cf - "$SOURCE_DIR" \ - | xz -c -9 \ - | gpg --batch --yes --cipher-algo AES256 --passphrase "$GPG_PASSPHRASE" -c \ - > "${ENCRYPTED_ARCHIVE_NAME}" +CHUNK_INDEX=1 +CURRENT_CHUNK_SIZE=0 +TMP_FILELIST=$(mktemp) -echo "=== Step 2: Split the encrypted archive into $CHUNK_SIZE chunks ===" -split -b "$CHUNK_SIZE" -a 3 "${ENCRYPTED_ARCHIVE_NAME}" "${ENCRYPTED_ARCHIVE_NAME}." +function start_new_chunk() { + # We'll reset the chunk accumulators + rm -f "$TMP_FILELIST" + touch "$TMP_FILELIST" + CURRENT_CHUNK_SIZE=0 +} -# Remove the single large file to save space (optional). -rm -f "${ENCRYPTED_ARCHIVE_NAME}" +# Initialize chunk +start_new_chunk -echo "=== Step 3: Generate checksums and a manifest/catalog ===" -touch "${MANIFEST_NAME}" +while read -r line; do + FILE_SIZE=$(echo "$line" | awk '{print $1}') + FILE_PATH=$(echo "$line" | cut -d' ' -f2-) -echo "Backup Manifest - $(date)" >> "${MANIFEST_NAME}" -echo "Source directory: $SOURCE_DIR" >> "${MANIFEST_NAME}" -echo "Destination directory: $DEST_DIR" >> "${MANIFEST_NAME}" -echo "Split chunk size: $CHUNK_SIZE" >> "${MANIFEST_NAME}" -echo "Encrypted archive chunk names:" >> "${MANIFEST_NAME}" -echo >> "${MANIFEST_NAME}" + # If adding this file exceeds chunk size, finalize the current chunk first + if [[ $(( CURRENT_CHUNK_SIZE + FILE_SIZE )) -gt $(( $(numfmt --from=iec $CHUNK_SIZE) )) ]]; then + # Finalize the chunk + # 1) Tar all the files in TMP_FILELIST + # 2) Compress with lz4 + # 3) Encrypt with gpg + # 4) Output a .tar.lz4.gpg in WORK_DIR + CHUNK_NAME=$(printf "chunk_%03d.tar.lz4.gpg" $CHUNK_INDEX) -for chunk in ${ENCRYPTED_ARCHIVE_NAME}.*; do - CHUNK_SHA256=$(compute_sha256 "$chunk") - echo "$CHUNK_SHA256" >> "${MANIFEST_NAME}" -done + echo "==> Creating chunk #$CHUNK_INDEX with the collected files..." + tar -cf - -T "$TMP_FILELIST" \ + | lz4 -c \ + | gpg --batch --yes --cipher-algo AES256 --passphrase "$GPG_PASSPHRASE" -c \ + > "${WORK_DIR}/${CHUNK_NAME}" -echo "Manifest created at: ${WORK_DIR}/${MANIFEST_NAME}" + # Compute checksum & record + CHUNK_SHA256=$(compute_sha256 "${WORK_DIR}/${CHUNK_NAME}") + echo "Chunk #$CHUNK_INDEX -> ${CHUNK_NAME}" >> "$MANIFEST_FILE" + echo "$CHUNK_SHA256" >> "$MANIFEST_FILE" + echo >> "$MANIFEST_FILE" -# If ISO creation is requested -if [ "$CREATE_ISO" = true ]; then - echo "=== Step 4: Create an ISO for each chunk (for easier burning) ===" - - # We'll place ISOs in a subfolder - mkdir -p iso_chunks - - for chunk in ${ENCRYPTED_ARCHIVE_NAME}.*; do - ISO_BASENAME="${chunk}.iso" - - # Create a temporary directory to hold the chunk file - mkdir -p temp_dir - cp "$chunk" temp_dir/ - - # Build an ISO with a single file inside: - genisoimage -o "iso_chunks/${ISO_BASENAME}" -V "ENCRYPTED_BACKUP" temp_dir >/dev/null 2>&1 || \ - mkisofs -o "iso_chunks/${ISO_BASENAME}" -V "ENCRYPTED_BACKUP" temp_dir - - # Remove the temporary directory - rm -rf temp_dir - done - - echo "ISO files created under: ${WORK_DIR}/iso_chunks" -fi - -# If burning is requested, attempt to burn right away. -# For cross-platform compatibility, we'll provide examples. -# You may need to adapt device names (/dev/sr0, /dev/dvd, etc.). - -if [ "$BURN_MEDIA" = true ]; then - echo "=== Step 5: Burn chunks/ISOs to M-Disc ===" - - # Example using growisofs on Linux: - # growisofs -Z /dev/sr0=chunk_or_iso - # or: - # growisofs -use-the-force-luke=dao -speed=2 -Z /dev/sr0=chunk_or_iso - - # Example using hdiutil on macOS for ISO: - # hdiutil burn chunk.iso - - echo "Attempting to burn each chunk (or ISO) to M-Disc. Please ensure a blank M-Disc is loaded each time." - - if [ "$CREATE_ISO" = true ]; then - # We have ISO images - for iso_file in iso_chunks/*.iso; do - echo "Insert new disc for: $iso_file" - read -p "Press [Enter] when ready to burn..." - - # Linux (growisofs) example: - if command -v growisofs >/dev/null 2>&1; then - growisofs -Z /dev/sr0="$iso_file" - elif [[ "$OSTYPE" == "darwin"* ]]; then - # macOS example using hdiutil - hdiutil burn "$iso_file" - else - echo "No known burner command found. Please burn manually: $iso_file" - fi - done - else - # Burn the chunk files directly. - for chunk in ${ENCRYPTED_ARCHIVE_NAME}.*; do - echo "Insert new disc for: $chunk" - read -p "Press [Enter] when ready to burn..." - - if command -v growisofs >/dev/null 2>&1; then - growisofs -Z /dev/sr0="$chunk" - elif [[ "$OSTYPE" == "darwin"* ]]; then - # We can't directly burn a raw file with hdiutil. Typically you'd create an ISO first. - # So warn the user. - echo "On macOS, please create an ISO or use a separate burning tool for: $chunk" - else - echo "No known burner command found. Please burn manually: $chunk" - fi - done + ((CHUNK_INDEX++)) + start_new_chunk fi - echo "Burning process completed. Verify your discs for peace of mind." + # Add current file to the chunk + echo "$FILE_PATH" >> "$TMP_FILELIST" + CURRENT_CHUNK_SIZE=$(( CURRENT_CHUNK_SIZE + FILE_SIZE )) +done < "$FILE_LIST" + +# If TMP_FILELIST still has leftover files, finalize the last chunk +LAST_LIST_SIZE=$(wc -l < "$TMP_FILELIST") +if [[ "$LAST_LIST_SIZE" -gt 0 ]]; then + CHUNK_NAME=$(printf "chunk_%03d.tar.lz4.gpg" $CHUNK_INDEX) + echo "==> Creating final chunk #$CHUNK_INDEX..." + tar -cf - -T "$TMP_FILELIST" \ + | lz4 -c \ + | gpg --batch --yes --cipher-algo AES256 --passphrase "$GPG_PASSPHRASE" -c \ + > "${WORK_DIR}/${CHUNK_NAME}" + + # Compute checksum & record + CHUNK_SHA256=$(compute_sha256 "${WORK_DIR}/${CHUNK_NAME}") + echo "Chunk #$CHUNK_INDEX -> ${CHUNK_NAME}" >> "$MANIFEST_FILE" + echo "$CHUNK_SHA256" >> "$MANIFEST_FILE" + echo >> "$MANIFEST_FILE" fi -echo "=== All done! ===" -echo "Your backup chunks and manifest are in: ${WORK_DIR}" -echo "Keep the manifest safe. You'll need all chunk files + passphrase to restore." +echo "=== All Chunks Created ===" +echo "Chunks and manifest are located in: $WORK_DIR" +echo "Manifest file: $MANIFEST_FILE" + +# Cleanup +rm -f "$FILE_LIST" "$TMP_FILELIST" + exit 0 \ No newline at end of file From df214ea14c70dc52477549b3ffdb759b5473d5ef Mon Sep 17 00:00:00 2001 From: first Date: Fri, 24 Jan 2025 06:51:35 +0000 Subject: [PATCH 2/5] Update README.md Signed-off-by: first --- README.md | 181 +++++++++++++++++++++++++++++++----------------------- 1 file changed, 103 insertions(+), 78 deletions(-) diff --git a/README.md b/README.md index f9c880a..599c34c 100644 --- a/README.md +++ b/README.md @@ -1,100 +1,125 @@ -Below is a sample Bash script and accompanying guide that demonstrate one way to automate: +# 1. Overview: Individually Encrypted 100GB Archives -1. **Creating a single encrypted archive** from your data. -2. **Splitting** that encrypted archive into 100GB chunks. -3. **Generating checksums** and a manifest/catalog. -4. **Optionally creating ISO images** from each chunk for more convenient burning. -5. **Burning** the resulting chunks (or ISOs) to M-Disc. +### Basic Idea -> **Important** -> - This script is written in Bash for Linux/macOS compatibility. It should also work on FreeBSD with minimal (if any) modifications, but you may need to install or adjust the relevant tools. -> - The script focuses on automating the encryption and splitting steps, as well as generating a manifest. -> - Burning to M-Disc on different platforms can vary. We show an example using `growisofs` (common on Linux) and `hdiutil` (macOS). Adjust as needed. -> - For best security, do **not** hardcode your passphrase in the script. You should be prompted for it. +1. **Figure out which files belong to which 100GB set**. + - You can gather files until their combined uncompressed size is ~100GB (or 95GB if you want some buffer for overhead). + - Put them in a "chunk_001" grouping, then "chunk_002," etc. + +2. **Create a TAR for each group**, then **compress** with `lz4`, then **encrypt** with `gpg`. + - Result: `chunk_001.tar.lz4.gpg`, `chunk_002.tar.lz4.gpg`, etc. + - Each chunk is fully independent: if you only have `chunk_004.tar.lz4.gpg`, you can decrypt it, decompress it, and restore the files that were in chunk #4. + +3. **Burn each chunk** onto its own M-Disc. + - Optionally, create ISO images (e.g., `genisoimage -o chunk_001.iso chunk_001.tar.lz4.gpg`) and then burn them. + +4. **To restore** any subset, you just decrypt the chunk you want, decompress, and extract it. No other chunks are required. + +### Pros + +- Each 100GB chunk is an autonomous backup. +- Damage/loss of one disc only affects that chunk's files. + +### Cons + +- Less efficient if you have many smaller files (no cross-chunk deduplication). +- Slightly more complex to create "balanced" 100GB sets. +- Big single files that exceed 100GB are a problem unless you handle them specially. --- -## How to Use the Script +# 2. Sample Script: `backup2mdisc.sh` -1. **Install Dependencies** - Make sure the following tools are installed on your system(s): - - **tar** - - **xz** - - **gpg** - - **split** - - **sha256sum** (or `shasum` on FreeBSD/macOS) - - **genisoimage** or **mkisofs** (for creating ISOs if desired) - - **growisofs** (Linux) or **hdiutil** (macOS) for burning. +This is a **Bash** script that: -2. **Make the Script Executable** - ```bash - chmod +x backup2mdisc.sh - ``` +1. Collects **all files** in a specified source directory. +2. Iterates over them in ascending order by size (you can adjust if you prefer a different approach). +3. Accumulates files into a "chunk" until you're about to exceed the chunk size limit. +4. When the chunk is "full," it creates a **tar** archive, pipes it into **lz4**, then **encrypts** with `gpg`. +5. Moves on to the next chunk until all files are processed. +6. Generates a manifest with checksums for each `.tar.lz4.gpg`. -3. **Run the Script** - ```bash - ./backup2mdisc.sh /path/to/source /path/to/destination 100G --create-iso --burn - ``` - - **`/path/to/source`**: The directory you want to back up. - - **`/path/to/destination`**: Where to store the intermediate backup files before burning. - - **`100G`**: The chunk size. Adjust if you're using different capacity discs. - - **`--create-iso`** (optional): Create ISO images from each chunk for more convenient burning. - - **`--burn`** (optional): Attempt to burn each chunk/ISO to disc automatically. - -4. **Enter Your GPG Passphrase** - - The script will prompt for a passphrase. This passphrase encrypts your data. Keep it safe! - -5. **Wait for the Script to Finish** - - A large `tar` + `xz` + `gpg` pipeline can take a considerable amount of time depending on your data size. - - After encryption, it splits into 100GB chunks. - - It then generates a **manifest** with SHA-256 checksums of each chunk. - -6. **Burn to M-Disc** - - If you used `--burn`, the script will prompt you to insert an M-Disc for each chunk or ISO. - - On Linux, it uses `growisofs`. On macOS, it attempts `hdiutil` if ISO files exist. - - If you prefer manual burning, skip `--burn` and burn the `.iso` files using your favorite tool. - -7. **Store the Manifest Safely** - - The manifest (`backup_manifest.txt`) in the work directory includes: - - Checksums for each chunk. - - The original source path. - - Timestamp. - - Keep this manifest (and the passphrase!) somewhere secure. You'll need all parts to restore. +> **Disclaimer**: +> - This script uses file-size-based grouping. If you have one single file larger than the chunk limit, it won't fit. You'd need advanced splitting or a different solution. +> - On macOS or FreeBSD, you might need to install or alias `sha256sum`. If unavailable, replace with `shasum -a 256`. +> - This script **does not** automatically burn discs (though it shows how you might add that step). --- -## Restoring Your Backup +## How This Script Works -To **restore** from these discs: +1. **Collect Files and Sort** + - We use `find` to list all files in `SOURCE_DIR`, capturing both size and path. + - Sorting by size ensures the script packs smaller files first. (You can remove sorting if you prefer alphabetical or another method.) -1. Copy all chunk files (or `.iso` contents) back to a working directory on your system. -2. Combine them back into a single file (if they were split outside of an ISO filesystem, just `cat` them together): - ```bash - cat backup.tar.xz.gpg.* > backup.tar.xz.gpg - ``` -3. Decrypt and extract: - ```bash - gpg --decrypt backup.tar.xz.gpg | xz -d | tar -xvf - - ``` - You'll be prompted for the same GPG passphrase. Once it's done, the original files/folders should appear in your current directory. +2. **Accumulate Files Until the Chunk Is ~100GB** + - We convert `CHUNK_SIZE` from something like `100G` into bytes. Then we compare the sum of file sizes to that limit. + - If adding a new file would exceed the chunk limit, we finalize the current chunk and create a new one. + +3. **Create a TAR, Compress with lz4, Then Encrypt** + - We pipe the TAR stream into `lz4` for fast compression, and then pipe **that** into `gpg --batch -c` for symmetric encryption with AES256. + - Each chunk is written to `chunk_XXX.tar.lz4.gpg`. + - No chunk depends on the others. + +4. **Write Checksums to the Manifest** + - We run a SHA-256 on the resulting `chunk_XXX.tar.lz4.gpg` and store that in `manifest_individual_chunks.txt` for integrity checks. + +5. **Repeat** + - Next chunk continues until all files have been processed. + +6. **Result** + - You get multiple `.tar.lz4.gpg` archives in your `DEST_DIR`, each below your chosen chunk size and fully independent. + +## Burning to M-Disc + +You can then burn each chunk to a separate disc. For example: + +```bash +cd /path/to/work_dir +genisoimage -o chunk_001.iso chunk_001.tar.lz4.gpg +# Then burn chunk_001.iso +growisofs -Z /dev/sr0=chunk_001.iso +``` + +Repeat for each chunk. On macOS, you might use: + +```bash +hdiutil burn chunk_001.iso +``` + +(Adjust device paths and commands as needed.) + +## Restoring Data + +To restore from a single chunk (e.g., chunk_002.tar.lz4.gpg), do: + +```bash +gpg --decrypt chunk_002.tar.lz4.gpg | lz4 -d | tar -xvf - +``` + +You'll be prompted for the same passphrase you used when creating the archive. After extraction, you'll see all the files that chunk contained. + +- **If one disc is lost**, you can still decrypt and restore the other discs. You only lose the files in the missing chunk. --- -### Notes & Tips +# Why lz4 Over xz? -- **Individual Chunk Decryption**: - The above script creates **one** large encrypted archive, then splits it. You need **all** parts to decrypt. If you want each 100GB chunk to be decryptable separately, you'd need to tar smaller subsets of data individually, encrypt each, and then burn. This is more complex and requires advanced scripting or a specialized backup tool. +- **lz4** is extremely fast compared to xz, especially for decompression. +- **xz** typically yields better compression (smaller output size), but at a much higher CPU cost. +- For backups where speed is the priority (and you have enough disc space), lz4 is a great choice. +- If you need to cram as much data as possible into 100GB, you might prefer xz with a high compression setting—but your backup process and restoration would be slower. -- **Automated Backup Tools**: - You might also consider tools like **Duplicati**, **Borg**, or **restic**, which support encryption, deduplication, and chunking. However, writing those chunks onto M-Disc is still a manual step. +--- -- **Testing**: - Test with a small directory first (say 1GB) and 100MB “chunks” to ensure your workflow is correct. Then proceed to the full data. +## Final Thoughts -- **M-Disc Drive Compatibility**: - Make sure your optical drive explicitly supports writing to 100GB BD-XL M-Disc. Standard Blu-ray or DVD drives often do not support higher-capacity M-Discs. +With this script and approach: -- **Verification**: - Always verify that your burned discs are readable. You can mount them and use the checksums from the manifest to confirm data integrity. +- You gain **independently decryptable** 100GB archives. +- If a single disc is damaged, you only lose that chunk's data; all other chunks remain fully restorable. +- lz4 + gpg is a solid combo for speed (lz4 for compression, gpg for encryption). +- Always **test** your workflow on smaller data sets before doing a large 2TB backup. +- Keep your passphrase secure, and consider verifying your burned discs with checksums. -That's it! This script and guide should get you started creating encrypted backups on 100GB M-Discs, with a manifest to track chunks and checksums, plus optional ISO creation and automated burning steps. Adjust as necessary for your specific environment and needs. \ No newline at end of file +That's it! You now have a **fast, chunked, and individually encrypted** backup solution for your M-Discs. \ No newline at end of file From 7ff6fd9a75b525f2e915025e9378fc3d445a52be Mon Sep 17 00:00:00 2001 From: first Date: Fri, 24 Jan 2025 07:17:34 +0000 Subject: [PATCH 3/5] re-add burn and create-iso Signed-off-by: first --- backup2mdisc.sh | 296 ++++++++++++++++++++++++++++++++---------------- 1 file changed, 199 insertions(+), 97 deletions(-) diff --git a/backup2mdisc.sh b/backup2mdisc.sh index bc41de8..9891279 100644 --- a/backup2mdisc.sh +++ b/backup2mdisc.sh @@ -3,175 +3,277 @@ # backup2mdisc.sh # # Purpose: -# Creates multiple self-contained 100GB (default) backup archives, each encrypted -# independently. Useful for writing to large-capacity M-Discs where you want -# each disc to be decryptable on its own. +# 1. Scans all files in a source directory. +# 2. Groups them into "chunks" so that each chunk is <= a specified size (default 100GB). +# 3. Creates a TAR archive of each chunk, compresses it with lz4, and encrypts it with GPG (AES256). +# 4. Each .tar.lz4.gpg is fully independent (no other parts/discs needed to restore that chunk). +# 5. (Optional) Creates ISO images from each encrypted chunk if --create-iso is provided. +# 6. (Optional) Burns each chunk or ISO to M-Disc if --burn is provided. # -# Requirements: +# Usage: +# ./backup2mdisc.sh /path/to/source /path/to/destination [CHUNK_SIZE] [--create-iso] [--burn] +# +# Examples: +# ./backup2mdisc.sh /home/user/data /mnt/backup 100G --create-iso +# ./backup2mdisc.sh /data /backup 50G --burn +# +# Dependencies: # - bash # - gpg (for encryption) # - lz4 (for fast compression) # - tar +# - split or file-based grouping approach # - sha256sum (or 'shasum -a 256' on macOS/FreeBSD) +# - genisoimage or mkisofs (for creating ISOs if --create-iso) +# - growisofs (Linux) or hdiutil (macOS) for burning if --burn # -# Usage: -# ./backup2mdisc.sh /path/to/source /path/to/destination [chunk_size] -# -# Example: -# ./backup2mdisc.sh /home/user/documents /mnt/backup 100G -# -# Tips: -# - If you want to burn these archives to disc afterward, you can: -# genisoimage -o chunk_001.iso chunk_001.tar.lz4.gpg -# Then burn the ISO using growisofs or hdiutil, etc. -# -# - Each chunk is standalone. If chunk #3 is lost, the rest are unaffected, -# but you lose only the files on chunk #3. -# -# - If you have a file larger than 'chunk_size', this script won't handle it -# gracefully. You'd need to adjust or handle large files differently. +# Notes: +# - This script sorts files by size and accumulates them until the chunk is "full." +# - If a file alone is bigger than CHUNK_SIZE, this script won't handle it gracefully. +# - Each chunk gets a separate .tar.lz4.gpg file. If one disc is lost, only that chunk's files are lost. +# - Keep your GPG passphrase safe; you'll need it to decrypt any chunk. # set -e -# Default chunk size -DEFAULT_CHUNK_SIZE="100G" - ##################################### -# HELPER FUNCTIONS # +# CONFIGURATION & DEFAULTS # ##################################### +DEFAULT_CHUNK_SIZE="100G" # Adjust if you want a different default +MANIFEST_NAME="manifest_individual_chunks.txt" + +##################################### +# FUNCTIONS # +##################################### + +function usage() { + echo "Usage: $0 /path/to/source /path/to/destination [CHUNK_SIZE] [--create-iso] [--burn]" + echo + echo "Example: $0 /home/user/docs /mnt/backup 100G --create-iso --burn" + exit 1 +} + +# Cross-platform SHA-256 function compute_sha256() { if command -v sha256sum >/dev/null 2>&1; then sha256sum "$1" else - # macOS/FreeBSD fallback: shasum -a 256 "$1" fi } -function usage() { - echo "Usage: $0 /path/to/source /path/to/destination [chunk_size]" - echo "Example: $0 /data /backup 100G" - exit 1 -} - ##################################### -# MAIN PROGRAM # +# MAIN SCRIPT # ##################################### -# Parse arguments +# Parse primary arguments SOURCE_DIR="$1" DEST_DIR="$2" CHUNK_SIZE="${3:-$DEFAULT_CHUNK_SIZE}" +# Shift away the first 3 arguments if present +shift 3 || true + +CREATE_ISO=false +BURN_MEDIA=false + +# Parse flags +for arg in "$@"; do + case "$arg" in + --create-iso) + CREATE_ISO=true + ;; + --burn) + BURN_MEDIA=true + ;; + *) + ;; + esac +done + +# Basic checks if [[ -z "$SOURCE_DIR" || -z "$DEST_DIR" ]]; then usage fi if [[ ! -d "$SOURCE_DIR" ]]; then - echo "ERROR: Source directory does not exist: $SOURCE_DIR" + echo "ERROR: Source directory '$SOURCE_DIR' does not exist." exit 1 fi if [[ ! -d "$DEST_DIR" ]]; then - echo "ERROR: Destination directory does not exist: $DEST_DIR" + echo "ERROR: Destination directory '$DEST_DIR' does not exist." exit 1 fi -# Prompt for GPG passphrase (do not store in script) +# Prompt for GPG passphrase echo -n "Enter GPG passphrase (will not be displayed): " read -s GPG_PASSPHRASE echo -# Create a working subdir +# Create a working directory WORK_DIR="${DEST_DIR}/individual_chunks_$(date +%Y%m%d_%H%M%S)" mkdir -p "$WORK_DIR" -# This file will track which files are in which chunk, plus checksums -MANIFEST_FILE="${WORK_DIR}/manifest_individual_chunks.txt" +# Create a manifest file to track chunk -> files mapping and checksums +MANIFEST_FILE="${WORK_DIR}/${MANIFEST_NAME}" touch "$MANIFEST_FILE" -echo "Manifest for individual-chunk backup" > "$MANIFEST_FILE" +echo "Manifest for independent chunks backup" > "$MANIFEST_FILE" echo "Source: $SOURCE_DIR" >> "$MANIFEST_FILE" echo "Timestamp: $(date)" >> "$MANIFEST_FILE" -echo "Chunk size: $CHUNK_SIZE" >> "$MANIFEST_FILE" +echo "Chunk size limit: $CHUNK_SIZE" >> "$MANIFEST_FILE" echo >> "$MANIFEST_FILE" -# List of all files with size, sorted by file size ascending -# If you prefer alphabetical, remove the "-printf '%s %p\n'| sort -n" logic -FILE_LIST=$(mktemp) -find "$SOURCE_DIR" -type f -printf "%s %p\n" | sort -n > "$FILE_LIST" +# Step 1: Collect all files with their sizes and sort them (ascending by size). +TEMP_FILE_LIST=$(mktemp) +find "$SOURCE_DIR" -type f -printf "%s %p\n" | sort -n > "$TEMP_FILE_LIST" CHUNK_INDEX=1 CURRENT_CHUNK_SIZE=0 -TMP_FILELIST=$(mktemp) +TMP_CHUNK_LIST=$(mktemp) + +function bytes_from_iec() { + # Convert something like '100G' or '50G' into bytes using numfmt + numfmt --from=iec "$1" +} + +MAX_CHUNK_BYTES=$(bytes_from_iec "$CHUNK_SIZE") function start_new_chunk() { - # We'll reset the chunk accumulators - rm -f "$TMP_FILELIST" - touch "$TMP_FILELIST" + rm -f "$TMP_CHUNK_LIST" + touch "$TMP_CHUNK_LIST" CURRENT_CHUNK_SIZE=0 } -# Initialize chunk +function finalize_chunk() { + # Called when we have a list of files in TMP_CHUNK_LIST and we want to + # 1) TAR them + # 2) Compress with lz4 + # 3) Encrypt with GPG + # 4) Possibly create ISO + # 5) Possibly burn + # 6) Update manifest + + local chunk_name + chunk_name=$(printf "chunk_%03d.tar.lz4.gpg" "$CHUNK_INDEX") + + echo + echo "==> Creating chunk #$CHUNK_INDEX: $chunk_name" + + # Tar + lz4 + gpg pipeline + tar -cf - -T "$TMP_CHUNK_LIST" \ + | lz4 -c \ + | gpg --batch --yes --cipher-algo AES256 --passphrase "$GPG_PASSPHRASE" -c \ + > "${WORK_DIR}/${chunk_name}" + + # Generate a SHA-256 sum + local chunk_path="${WORK_DIR}/${chunk_name}" + local sum_line + sum_line=$(compute_sha256 "$chunk_path") + + # Add chunk info to manifest + echo "Chunk #$CHUNK_INDEX -> $chunk_name" >> "$MANIFEST_FILE" + echo "Files in this chunk:" >> "$MANIFEST_FILE" + cat "$TMP_CHUNK_LIST" >> "$MANIFEST_FILE" + echo "" >> "$MANIFEST_FILE" + echo "SHA256: $sum_line" >> "$MANIFEST_FILE" + echo "-----------------------------------" >> "$MANIFEST_FILE" + echo >> "$MANIFEST_FILE" + + # Optionally create ISO + local iso_name + iso_name=$(printf "chunk_%03d.iso" "$CHUNK_INDEX") + if [ "$CREATE_ISO" = true ]; then + echo "==> Creating ISO for chunk #$CHUNK_INDEX" + mkdir -p "${WORK_DIR}/iso_chunks" + local temp_iso_dir="${WORK_DIR}/temp_iso_dir_$CHUNK_INDEX" + mkdir -p "$temp_iso_dir" + + # Copy the encrypted archive into a temp directory + cp "$chunk_path" "$temp_iso_dir"/ + + # Build the ISO + local iso_output="${WORK_DIR}/iso_chunks/${iso_name}" + if command -v genisoimage >/dev/null 2>&1; then + genisoimage -quiet -o "$iso_output" -V "ENCRYPTED_BACKUP_${CHUNK_INDEX}" "$temp_iso_dir" + else + # Try mkisofs + mkisofs -quiet -o "$iso_output" -V "ENCRYPTED_BACKUP_${CHUNK_INDEX}" "$temp_iso_dir" + fi + rm -rf "$temp_iso_dir" + + # If --burn is also requested, burn the ISO + if [ "$BURN_MEDIA" = true ]; then + echo + echo "Please insert a blank M-Disc for chunk #$CHUNK_INDEX (ISO): $iso_name" + read -rp "Press [Enter] when ready to burn..." + if command -v growisofs >/dev/null 2>&1; then + growisofs -Z /dev/sr0="$iso_output" + elif [[ "$OSTYPE" == "darwin"* ]]; then + # macOS example + hdiutil burn "$iso_output" + else + echo "No recognized burner found. Please burn ${iso_output} manually." + fi + fi + else + # If we are not creating ISO but we are burning the chunk file directly + if [ "$BURN_MEDIA" = true ]; then + echo + echo "Please insert a blank M-Disc for chunk #$CHUNK_INDEX: $chunk_name" + read -rp "Press [Enter] when ready to burn..." + if command -v growisofs >/dev/null 2>&1; then + growisofs -Z /dev/sr0="$chunk_path" + elif [[ "$OSTYPE" == "darwin"* ]]; then + # hdiutil doesn't burn a raw file easily, typically it expects .iso + echo "On macOS, consider creating an ISO or using a different burning tool for $chunk_name." + else + echo "No recognized burner found. Please burn ${chunk_path} manually." + fi + fi + fi + + ((CHUNK_INDEX++)) + start_new_chunk +} + +# Initialize the first chunk start_new_chunk -while read -r line; do +# Step 2: Go through each file, add to chunk if it fits, otherwise finalize and start a new chunk. +while IFS= read -r line; do FILE_SIZE=$(echo "$line" | awk '{print $1}') FILE_PATH=$(echo "$line" | cut -d' ' -f2-) - # If adding this file exceeds chunk size, finalize the current chunk first - if [[ $(( CURRENT_CHUNK_SIZE + FILE_SIZE )) -gt $(( $(numfmt --from=iec $CHUNK_SIZE) )) ]]; then - # Finalize the chunk - # 1) Tar all the files in TMP_FILELIST - # 2) Compress with lz4 - # 3) Encrypt with gpg - # 4) Output a .tar.lz4.gpg in WORK_DIR - CHUNK_NAME=$(printf "chunk_%03d.tar.lz4.gpg" $CHUNK_INDEX) - - echo "==> Creating chunk #$CHUNK_INDEX with the collected files..." - tar -cf - -T "$TMP_FILELIST" \ - | lz4 -c \ - | gpg --batch --yes --cipher-algo AES256 --passphrase "$GPG_PASSPHRASE" -c \ - > "${WORK_DIR}/${CHUNK_NAME}" - - # Compute checksum & record - CHUNK_SHA256=$(compute_sha256 "${WORK_DIR}/${CHUNK_NAME}") - echo "Chunk #$CHUNK_INDEX -> ${CHUNK_NAME}" >> "$MANIFEST_FILE" - echo "$CHUNK_SHA256" >> "$MANIFEST_FILE" - echo >> "$MANIFEST_FILE" - - ((CHUNK_INDEX++)) - start_new_chunk + # If adding this file exceeds the chunk limit, finalize the current chunk now + if [[ $((CURRENT_CHUNK_SIZE + FILE_SIZE)) -gt $MAX_CHUNK_BYTES ]]; then + # Finalize current chunk if it has at least 1 file + if [[ $(wc -l < "$TMP_CHUNK_LIST") -gt 0 ]]; then + finalize_chunk + fi fi - # Add current file to the chunk - echo "$FILE_PATH" >> "$TMP_FILELIST" - CURRENT_CHUNK_SIZE=$(( CURRENT_CHUNK_SIZE + FILE_SIZE )) -done < "$FILE_LIST" + # Add the file to the chunk + echo "$FILE_PATH" >> "$TMP_CHUNK_LIST" + CURRENT_CHUNK_SIZE=$((CURRENT_CHUNK_SIZE + FILE_SIZE)) +done < "$TEMP_FILE_LIST" -# If TMP_FILELIST still has leftover files, finalize the last chunk -LAST_LIST_SIZE=$(wc -l < "$TMP_FILELIST") -if [[ "$LAST_LIST_SIZE" -gt 0 ]]; then - CHUNK_NAME=$(printf "chunk_%03d.tar.lz4.gpg" $CHUNK_INDEX) - echo "==> Creating final chunk #$CHUNK_INDEX..." - tar -cf - -T "$TMP_FILELIST" \ - | lz4 -c \ - | gpg --batch --yes --cipher-algo AES256 --passphrase "$GPG_PASSPHRASE" -c \ - > "${WORK_DIR}/${CHUNK_NAME}" - - # Compute checksum & record - CHUNK_SHA256=$(compute_sha256 "${WORK_DIR}/${CHUNK_NAME}") - echo "Chunk #$CHUNK_INDEX -> ${CHUNK_NAME}" >> "$MANIFEST_FILE" - echo "$CHUNK_SHA256" >> "$MANIFEST_FILE" - echo >> "$MANIFEST_FILE" +# Finalize the last chunk if it has leftover files +if [[ $(wc -l < "$TMP_CHUNK_LIST") -gt 0 ]]; then + finalize_chunk fi -echo "=== All Chunks Created ===" -echo "Chunks and manifest are located in: $WORK_DIR" -echo "Manifest file: $MANIFEST_FILE" +echo +echo "=== All chunks created ===" +echo "Your chunks (and possibly ISOs) are located in:" +echo " $WORK_DIR" +echo +echo "Manifest: $MANIFEST_FILE" +echo "-----------------------------------" +echo "Done!" # Cleanup -rm -f "$FILE_LIST" "$TMP_FILELIST" +rm -f "$TEMP_FILE_LIST" "$TMP_CHUNK_LIST" exit 0 \ No newline at end of file From d1bfb381d443aa9b42c254796d668ce0e0539266 Mon Sep 17 00:00:00 2001 From: first Date: Fri, 24 Jan 2025 07:24:07 +0000 Subject: [PATCH 4/5] Update README.md Signed-off-by: first --- README.md | 158 ++++++++++++++++++++---------------------------------- 1 file changed, 57 insertions(+), 101 deletions(-) diff --git a/README.md b/README.md index 599c34c..1991253 100644 --- a/README.md +++ b/README.md @@ -1,125 +1,81 @@ -# 1. Overview: Individually Encrypted 100GB Archives +# backup2mdisc +## How It Works -### Basic Idea +1. **File Collection & Sorting** + - The script uses `find` to list all files in your `SOURCE_DIR` with their sizes. + - It sorts them in ascending order by size so it can pack smaller files first (you can remove `| sort -n` if you prefer a different method). -1. **Figure out which files belong to which 100GB set**. - - You can gather files until their combined uncompressed size is ~100GB (or 95GB if you want some buffer for overhead). - - Put them in a "chunk_001" grouping, then "chunk_002," etc. +2. **Chunk Accumulation** + - It iterates over each file, summing up file sizes into a “current chunk.” + - If adding a new file would exceed `CHUNK_SIZE` (default 100GB), it **finalizes** the current chunk (creates `.tar.lz4.gpg`) and starts a new one. -2. **Create a TAR for each group**, then **compress** with `lz4`, then **encrypt** with `gpg`. - - Result: `chunk_001.tar.lz4.gpg`, `chunk_002.tar.lz4.gpg`, etc. - - Each chunk is fully independent: if you only have `chunk_004.tar.lz4.gpg`, you can decrypt it, decompress it, and restore the files that were in chunk #4. +3. **Archive, Compress, Encrypt** + - For each chunk, it creates a `.tar.lz4.gpg` file. Specifically: + 1. `tar -cf - -T $TMP_CHUNK_LIST` (archive of the files in that chunk) + 2. Pipe into `lz4 -c` for fast compression + 3. Pipe into `gpg --batch -c` (symmetric encrypt with AES256, using your passphrase) + - The result is a self-contained file like `chunk_001.tar.lz4.gpg`. -3. **Burn each chunk** onto its own M-Disc. - - Optionally, create ISO images (e.g., `genisoimage -o chunk_001.iso chunk_001.tar.lz4.gpg`) and then burn them. +4. **Checksums & Manifest** + - It calculates the SHA-256 sum of each chunk archive and appends it to a manifest file along with the list of included files. + - That manifest is stored in `$WORK_DIR`. -4. **To restore** any subset, you just decrypt the chunk you want, decompress, and extract it. No other chunks are required. +5. **Optional ISO Creation** (`--create-iso`) + - After each chunk is created, the script can build an ISO image containing just that `.tar.lz4.gpg`. + - This step uses `genisoimage` (or `mkisofs`). The resulting file is `chunk_001.iso`, etc. -### Pros +6. **Optional Burning** (`--burn`) + - If you specify `--burn`, the script will pause after creating each chunk/ISO and prompt you to insert a fresh M-Disc. + - On **Linux**, it tries `growisofs`. + - On **macOS**, it tries `hdiutil` (if creating an ISO). + - If it doesn't find these commands, it'll instruct you to burn manually. -- Each 100GB chunk is an autonomous backup. -- Damage/loss of one disc only affects that chunk's files. - -### Cons - -- Less efficient if you have many smaller files (no cross-chunk deduplication). -- Slightly more complex to create "balanced" 100GB sets. -- Big single files that exceed 100GB are a problem unless you handle them specially. +7. **Repeat** + - The script loops until all files have been placed into chunk(s). --- -# 2. Sample Script: `backup2mdisc.sh` +## Restoring Your Data -This is a **Bash** script that: +- **Disc is self-contained**: If you have disc #4 containing `chunk_004.tar.lz4.gpg`, you can restore it independently of the others. +- **Decrypt & Extract**: + ```bash + gpg --decrypt chunk_004.tar.lz4.gpg | lz4 -d | tar -xvf - + ``` + This will prompt for the passphrase you used during backup. -1. Collects **all files** in a specified source directory. -2. Iterates over them in ascending order by size (you can adjust if you prefer a different approach). -3. Accumulates files into a "chunk" until you're about to exceed the chunk size limit. -4. When the chunk is "full," it creates a **tar** archive, pipes it into **lz4**, then **encrypts** with `gpg`. -5. Moves on to the next chunk until all files are processed. -6. Generates a manifest with checksums for each `.tar.lz4.gpg`. - -> **Disclaimer**: -> - This script uses file-size-based grouping. If you have one single file larger than the chunk limit, it won't fit. You'd need advanced splitting or a different solution. -> - On macOS or FreeBSD, you might need to install or alias `sha256sum`. If unavailable, replace with `shasum -a 256`. -> - This script **does not** automatically burn discs (though it shows how you might add that step). +- If one disc is lost, you only lose the files in that chunk; all other chunks remain restorable. --- -## How This Script Works +## Why lz4? -1. **Collect Files and Sort** - - We use `find` to list all files in `SOURCE_DIR`, capturing both size and path. - - Sorting by size ensures the script packs smaller files first. (You can remove sorting if you prefer alphabetical or another method.) - -2. **Accumulate Files Until the Chunk Is ~100GB** - - We convert `CHUNK_SIZE` from something like `100G` into bytes. Then we compare the sum of file sizes to that limit. - - If adding a new file would exceed the chunk limit, we finalize the current chunk and create a new one. - -3. **Create a TAR, Compress with lz4, Then Encrypt** - - We pipe the TAR stream into `lz4` for fast compression, and then pipe **that** into `gpg --batch -c` for symmetric encryption with AES256. - - Each chunk is written to `chunk_XXX.tar.lz4.gpg`. - - No chunk depends on the others. - -4. **Write Checksums to the Manifest** - - We run a SHA-256 on the resulting `chunk_XXX.tar.lz4.gpg` and store that in `manifest_individual_chunks.txt` for integrity checks. - -5. **Repeat** - - Next chunk continues until all files have been processed. - -6. **Result** - - You get multiple `.tar.lz4.gpg` archives in your `DEST_DIR`, each below your chosen chunk size and fully independent. - -## Burning to M-Disc - -You can then burn each chunk to a separate disc. For example: - -```bash -cd /path/to/work_dir -genisoimage -o chunk_001.iso chunk_001.tar.lz4.gpg -# Then burn chunk_001.iso -growisofs -Z /dev/sr0=chunk_001.iso -``` - -Repeat for each chunk. On macOS, you might use: - -```bash -hdiutil burn chunk_001.iso -``` - -(Adjust device paths and commands as needed.) - -## Restoring Data - -To restore from a single chunk (e.g., chunk_002.tar.lz4.gpg), do: - -```bash -gpg --decrypt chunk_002.tar.lz4.gpg | lz4 -d | tar -xvf - -``` - -You'll be prompted for the same passphrase you used when creating the archive. After extraction, you'll see all the files that chunk contained. - -- **If one disc is lost**, you can still decrypt and restore the other discs. You only lose the files in the missing chunk. +- **Speed**: `lz4` is extremely fast at both compression and decompression. +- **Less compression ratio** than xz, but if your priority is speed (and 100GB disc space is enough), `lz4` is a great choice. +- For maximum compression at the cost of time, you could replace `lz4` with `xz -9`, but expect slower backups and restores. --- -# Why lz4 Over xz? +## Tips & Caveats -- **lz4** is extremely fast compared to xz, especially for decompression. -- **xz** typically yields better compression (smaller output size), but at a much higher CPU cost. -- For backups where speed is the priority (and you have enough disc space), lz4 is a great choice. -- If you need to cram as much data as possible into 100GB, you might prefer xz with a high compression setting—but your backup process and restoration would be slower. +1. **Large Files** + - A single file larger than your chunk size (e.g., 101GB file with a 100GB chunk limit) won't fit. This script doesn't handle that gracefully. You'd need to split such a file (e.g., with `split`) before archiving or use a backup tool that supports partial file splitting. + +2. **Verification** + - Always verify your discs after burning. Mount them and compare the chunk's SHA-256 with the manifest to ensure data integrity. + +3. **Incremental or Deduplicated Backups** + - For advanced features (incremental, deduplication, partial-chunk checksums), consider specialized backup programs (like Borg, restic, Duplicati). However, they usually produce multi-volume archives that need **all** volumes to restore. + +4. **Cross-Platform** + - On FreeBSD or macOS, you might need to tweak the commands for hashing (`sha256sum` vs. `shasum -a 256`) or ISO creation (`mkisofs` vs. `genisoimage`). + - For burning, Linux uses `growisofs`, macOS uses `hdiutil`, and FreeBSD may require `cdrecord` or another tool. --- -## Final Thoughts +**Now you can enjoy the best of both worlds**: +- **Independently decryptable** (and restorable) archives on each M-Disc. +- Automatic ISO creation and optional disc burning in the same script. +- Fast compression via lz4. -With this script and approach: - -- You gain **independently decryptable** 100GB archives. -- If a single disc is damaged, you only lose that chunk's data; all other chunks remain fully restorable. -- lz4 + gpg is a solid combo for speed (lz4 for compression, gpg for encryption). -- Always **test** your workflow on smaller data sets before doing a large 2TB backup. -- Keep your passphrase secure, and consider verifying your burned discs with checksums. - -That's it! You now have a **fast, chunked, and individually encrypted** backup solution for your M-Discs. \ No newline at end of file +This gives you a **self-contained** backup on each disc without chain-dependency across your entire 2TB backup set! \ No newline at end of file From 18b9a352901fb5b00ea08d787f8079317d50c029 Mon Sep 17 00:00:00 2001 From: first Date: Fri, 24 Jan 2025 07:30:57 +0000 Subject: [PATCH 5/5] add purpose, examples, depends Signed-off-by: first --- README.md | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/README.md b/README.md index 1991253..5b0ee7d 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,13 @@ # backup2mdisc + +## Purpose: +1. Scans all files in a source directory. +2. Groups them into "chunks" so that each chunk is <= a specified size (default 100GB). +3. Creates a TAR archive of each chunk, compresses it with `lz4`, and encrypts it with GPG (AES256). +4. Each `.tar.lz4.gpg` is fully independent (no other parts/discs needed to restore that chunk). +5. (Optional) Creates ISO images from each encrypted chunk if `--create-iso` is provided. +6. (Optional) Burns each chunk or ISO to M-Disc if `--burn` is provided. + ## How It Works 1. **File Collection & Sorting** @@ -35,6 +44,25 @@ --- +## Usage: +`./backup_in_individual_chunks.sh /path/to/source /path/to/destination [CHUNK_SIZE] [--create-iso] [--burn]` + +## Examples: +`./backup_in_individual_chunks.sh /home/user/data /mnt/backup 100G --create-iso` +`./backup_in_individual_chunks.sh /data /backup 50G --burn` + +## Dependencies: +- `bash` +- `gpg` (for encryption) +- `lz4` (for fast compression) +- `tar` +- `split` or file-based grouping approach +- `sha256sum` (or '`shasum -a 256`' on macOS/FreeBSD) +- `genisoimage` or `mkisofs` (for creating ISOs if `--create-iso`) +- `growisofs` (Linux) or `hdiutil` (macOS) for burning if `--burn` + +--- + ## Restoring Your Data - **Disc is self-contained**: If you have disc #4 containing `chunk_004.tar.lz4.gpg`, you can restore it independently of the others.