diff --git a/tools/copy-to-sha256.py b/tools/copy-to-sha256.py index ec12fb22..c82974dd 100755 --- a/tools/copy-to-sha256.py +++ b/tools/copy-to-sha256.py @@ -11,18 +11,6 @@ import sys import io HASH_LENGTH = 8 -USE_COMPRESSION = True - -if USE_COMPRESSION: - if sys.version_info >= (3, 14): - from compression import zstd - else: - try: - import zstandard as zstd - except ImportError: - print("Error: zstandard module required when USE_COMPRESSION = True") - print("Install with: pip install zstandard") - sys.exit(1) def hash_file(filename) -> str: with open(filename, "rb", buffering=0) as f: @@ -43,23 +31,39 @@ def main(): formatter_class=argparse.RawTextHelpFormatter) args.add_argument("from_path", metavar="from", help="from") args.add_argument("to_path", metavar="to", help="to") + args.add_argument("--zstd", action="store_true", help="Use Zstandard compression") args = args.parse_args() from_path = os.path.normpath(args.from_path) to_path = os.path.normpath(args.to_path) + # Import zstd only if compression is requested + zstd_module = None + if args.zstd: + if sys.version_info >= (3, 14): + from compression import zstd + zstd_module = zstd + else: + try: + import zstandard as zstd + zstd_module = zstd + except ImportError: + print("Error: zstandard module required when using --zstd flag") + print("Install with: pip install zstandard") + sys.exit(1) + if os.path.isfile(from_path): tar = tarfile.open(from_path, "r") else: tar = None if tar: - handle_tar(logger, tar, to_path) + handle_tar(logger, tar, to_path, args.zstd, zstd_module) else: - handle_dir(logger, from_path, to_path) + handle_dir(logger, from_path, to_path, args.zstd, zstd_module) -def handle_dir(logger, from_path: str, to_path: str): +def handle_dir(logger, from_path: str, to_path: str, use_compression: bool, zstd_module): def onerror(oserror): logger.warning(oserror) @@ -78,37 +82,37 @@ def handle_dir(logger, from_path: str, to_path: str): continue file_hash = hash_file(absname) - filename = file_hash[0:HASH_LENGTH] + (".bin.zst" if USE_COMPRESSION else ".bin") + filename = file_hash[0:HASH_LENGTH] + (".bin.zst" if use_compression else ".bin") to_abs = os.path.join(to_path, filename) if os.path.exists(to_abs): logger.info("Exists, skipped {} ({})".format(to_abs, absname)) else: - if USE_COMPRESSION: + if use_compression: logger.info("Compressing {} {}".format(absname, to_abs)) with open(absname, 'rb') as src_file: with open(to_abs, 'wb') as dst_file: - zstd.ZstdCompressor(level=3).copy_stream(src_file, dst_file) + zstd_module.ZstdCompressor(level=19).copy_stream(src_file, dst_file) else: logger.info("cp {} {}".format(absname, to_abs)) shutil.copyfile(absname, to_abs) -def handle_tar(logger, tar, to_path: str): +def handle_tar(logger, tar, to_path: str, use_compression: bool, zstd_module): for member in tar.getmembers(): if member.isfile() or member.islnk(): f = tar.extractfile(member) file_hash = hash_fileobj(f) - filename = file_hash[0:HASH_LENGTH] + (".bin.zst" if USE_COMPRESSION else ".bin") + filename = file_hash[0:HASH_LENGTH] + (".bin.zst" if use_compression else ".bin") to_abs = os.path.join(to_path, filename) if os.path.exists(to_abs): logger.info("Exists, skipped {} ({})".format(to_abs, member.name)) else: - if USE_COMPRESSION: + if use_compression: logger.info("Extracted and compressing {} ({})".format(to_abs, member.name)) f.seek(0) with open(to_abs, 'wb') as dst_file: - zstd.ZstdCompressor(level=3).copy_stream(f, dst_file) + zstd_module.ZstdCompressor(level=19).copy_stream(f, dst_file) else: logger.info("Extracted {} ({})".format(to_abs, member.name)) to_file = open(to_abs, "wb") diff --git a/tools/docker/alpine/build.sh b/tools/docker/alpine/build.sh index e6c4c7aa..72edfb8f 100755 --- a/tools/docker/alpine/build.sh +++ b/tools/docker/alpine/build.sh @@ -20,10 +20,10 @@ docker export "$CONTAINER_NAME" -o "$OUT_ROOTFS_TAR" # https://github.com/iximiuz/docker-to-linux/issues/19#issuecomment-1242809707 tar -f "$OUT_ROOTFS_TAR" --delete ".dockerenv" || true -../../../tools/fs2json.py --out "$OUT_FSJSON" "$OUT_ROOTFS_TAR" +../../../tools/fs2json.py --zstd --out "$OUT_FSJSON" "$OUT_ROOTFS_TAR" # Note: Not deleting old files here mkdir -p "$OUT_ROOTFS_FLAT" -../../../tools/copy-to-sha256.py "$OUT_ROOTFS_TAR" "$OUT_ROOTFS_FLAT" +../../../tools/copy-to-sha256.py --zstd "$OUT_ROOTFS_TAR" "$OUT_ROOTFS_FLAT" echo "$OUT_ROOTFS_TAR", "$OUT_ROOTFS_FLAT" and "$OUT_FSJSON" created. diff --git a/tools/fs2json.py b/tools/fs2json.py index 59d00bfa..2bf35aab 100755 --- a/tools/fs2json.py +++ b/tools/fs2json.py @@ -17,8 +17,6 @@ import tarfile VERSION = 3 -USE_COMPRESSION = True - IDX_NAME = 0 IDX_SIZE = 1 IDX_MTIME = 2 @@ -70,6 +68,8 @@ def main(): args.add_argument("path", metavar="path-or-tar", help="Base path or tar file to include in JSON") + args.add_argument("--zstd", action="store_true", + help="Use Zstandard compression") args = args.parse_args() @@ -81,9 +81,9 @@ def main(): tar = None if tar: - (root, total_size) = handle_tar(logger, tar) + (root, total_size) = handle_tar(logger, tar, args.zstd) else: - (root, total_size) = handle_dir(logger, path, args.exclude) + (root, total_size) = handle_dir(logger, path, args.exclude, args.zstd) if False: # normalize the order of children, useful to debug differences between @@ -105,7 +105,7 @@ def main(): logger.info("Creating json ...") json.dump(result, args.out, check_circular=False, separators=(',', ':')) -def handle_dir(logger, path, exclude): +def handle_dir(logger, path, exclude, use_compression): path = path + "/" exclude = exclude or [] exclude = [os.path.join("/", os.path.normpath(p)) for p in exclude] @@ -199,7 +199,7 @@ def handle_dir(logger, path, exclude): obj[IDX_TARGET] = target elif isfile: file_hash = hash_file(absname) - filename = file_hash[0:HASH_LENGTH] + (".bin.zst" if USE_COMPRESSION else ".bin") + filename = file_hash[0:HASH_LENGTH] + (".bin.zst" if use_compression else ".bin") existing = filename_to_hash.get(filename) assert existing is None or existing == file_hash, "Collision in short hash (%s and %s)" % (existing, file_hash) filename_to_hash[filename] = file_hash @@ -214,7 +214,7 @@ def handle_dir(logger, path, exclude): return (mainroot, total_size) -def handle_tar(logger, tar): +def handle_tar(logger, tar, use_compression): mainroot = [] filename_to_hash = {} total_size = 0 @@ -242,7 +242,7 @@ def handle_tar(logger, tar): obj[IDX_MODE] |= S_IFREG f = tar.extractfile(member) file_hash = hash_fileobj(f) - filename = file_hash[0:HASH_LENGTH] + (".bin.zst" if USE_COMPRESSION else ".bin") + filename = file_hash[0:HASH_LENGTH] + (".bin.zst" if use_compression else ".bin") existing = filename_to_hash.get(filename) assert existing is None or existing == file_hash, "Collision in short hash (%s and %s)" % (existing, file_hash) filename_to_hash[filename] = file_hash