mirror of
https://github.com/copy/v86.git
synced 2025-12-31 04:23:15 +00:00
Add --zstd flag and use 19 lvl compression
This commit is contained in:
Дмитрий Ценеков
Fabian
parent
bd493da2dc
commit
b91ad42fe2
3 changed files with 36 additions and 32 deletions
|
|
@ -11,18 +11,6 @@ import sys
|
|||
import io
|
||||
|
||||
HASH_LENGTH = 8
|
||||
USE_COMPRESSION = True
|
||||
|
||||
if USE_COMPRESSION:
|
||||
if sys.version_info >= (3, 14):
|
||||
from compression import zstd
|
||||
else:
|
||||
try:
|
||||
import zstandard as zstd
|
||||
except ImportError:
|
||||
print("Error: zstandard module required when USE_COMPRESSION = True")
|
||||
print("Install with: pip install zstandard")
|
||||
sys.exit(1)
|
||||
|
||||
def hash_file(filename) -> str:
|
||||
with open(filename, "rb", buffering=0) as f:
|
||||
|
|
@ -43,23 +31,39 @@ def main():
|
|||
formatter_class=argparse.RawTextHelpFormatter)
|
||||
args.add_argument("from_path", metavar="from", help="from")
|
||||
args.add_argument("to_path", metavar="to", help="to")
|
||||
args.add_argument("--zstd", action="store_true", help="Use Zstandard compression")
|
||||
|
||||
args = args.parse_args()
|
||||
|
||||
from_path = os.path.normpath(args.from_path)
|
||||
to_path = os.path.normpath(args.to_path)
|
||||
|
||||
# Import zstd only if compression is requested
|
||||
zstd_module = None
|
||||
if args.zstd:
|
||||
if sys.version_info >= (3, 14):
|
||||
from compression import zstd
|
||||
zstd_module = zstd
|
||||
else:
|
||||
try:
|
||||
import zstandard as zstd
|
||||
zstd_module = zstd
|
||||
except ImportError:
|
||||
print("Error: zstandard module required when using --zstd flag")
|
||||
print("Install with: pip install zstandard")
|
||||
sys.exit(1)
|
||||
|
||||
if os.path.isfile(from_path):
|
||||
tar = tarfile.open(from_path, "r")
|
||||
else:
|
||||
tar = None
|
||||
|
||||
if tar:
|
||||
handle_tar(logger, tar, to_path)
|
||||
handle_tar(logger, tar, to_path, args.zstd, zstd_module)
|
||||
else:
|
||||
handle_dir(logger, from_path, to_path)
|
||||
handle_dir(logger, from_path, to_path, args.zstd, zstd_module)
|
||||
|
||||
def handle_dir(logger, from_path: str, to_path: str):
|
||||
def handle_dir(logger, from_path: str, to_path: str, use_compression: bool, zstd_module):
|
||||
def onerror(oserror):
|
||||
logger.warning(oserror)
|
||||
|
||||
|
|
@ -78,37 +82,37 @@ def handle_dir(logger, from_path: str, to_path: str):
|
|||
continue
|
||||
|
||||
file_hash = hash_file(absname)
|
||||
filename = file_hash[0:HASH_LENGTH] + (".bin.zst" if USE_COMPRESSION else ".bin")
|
||||
filename = file_hash[0:HASH_LENGTH] + (".bin.zst" if use_compression else ".bin")
|
||||
to_abs = os.path.join(to_path, filename)
|
||||
|
||||
if os.path.exists(to_abs):
|
||||
logger.info("Exists, skipped {} ({})".format(to_abs, absname))
|
||||
else:
|
||||
if USE_COMPRESSION:
|
||||
if use_compression:
|
||||
logger.info("Compressing {} {}".format(absname, to_abs))
|
||||
with open(absname, 'rb') as src_file:
|
||||
with open(to_abs, 'wb') as dst_file:
|
||||
zstd.ZstdCompressor(level=3).copy_stream(src_file, dst_file)
|
||||
zstd_module.ZstdCompressor(level=19).copy_stream(src_file, dst_file)
|
||||
else:
|
||||
logger.info("cp {} {}".format(absname, to_abs))
|
||||
shutil.copyfile(absname, to_abs)
|
||||
|
||||
def handle_tar(logger, tar, to_path: str):
|
||||
def handle_tar(logger, tar, to_path: str, use_compression: bool, zstd_module):
|
||||
for member in tar.getmembers():
|
||||
if member.isfile() or member.islnk():
|
||||
f = tar.extractfile(member)
|
||||
file_hash = hash_fileobj(f)
|
||||
filename = file_hash[0:HASH_LENGTH] + (".bin.zst" if USE_COMPRESSION else ".bin")
|
||||
filename = file_hash[0:HASH_LENGTH] + (".bin.zst" if use_compression else ".bin")
|
||||
to_abs = os.path.join(to_path, filename)
|
||||
|
||||
if os.path.exists(to_abs):
|
||||
logger.info("Exists, skipped {} ({})".format(to_abs, member.name))
|
||||
else:
|
||||
if USE_COMPRESSION:
|
||||
if use_compression:
|
||||
logger.info("Extracted and compressing {} ({})".format(to_abs, member.name))
|
||||
f.seek(0)
|
||||
with open(to_abs, 'wb') as dst_file:
|
||||
zstd.ZstdCompressor(level=3).copy_stream(f, dst_file)
|
||||
zstd_module.ZstdCompressor(level=19).copy_stream(f, dst_file)
|
||||
else:
|
||||
logger.info("Extracted {} ({})".format(to_abs, member.name))
|
||||
to_file = open(to_abs, "wb")
|
||||
|
|
|
|||
|
|
@ -20,10 +20,10 @@ docker export "$CONTAINER_NAME" -o "$OUT_ROOTFS_TAR"
|
|||
# https://github.com/iximiuz/docker-to-linux/issues/19#issuecomment-1242809707
|
||||
tar -f "$OUT_ROOTFS_TAR" --delete ".dockerenv" || true
|
||||
|
||||
../../../tools/fs2json.py --out "$OUT_FSJSON" "$OUT_ROOTFS_TAR"
|
||||
../../../tools/fs2json.py --zstd --out "$OUT_FSJSON" "$OUT_ROOTFS_TAR"
|
||||
|
||||
# Note: Not deleting old files here
|
||||
mkdir -p "$OUT_ROOTFS_FLAT"
|
||||
../../../tools/copy-to-sha256.py "$OUT_ROOTFS_TAR" "$OUT_ROOTFS_FLAT"
|
||||
../../../tools/copy-to-sha256.py --zstd "$OUT_ROOTFS_TAR" "$OUT_ROOTFS_FLAT"
|
||||
|
||||
echo "$OUT_ROOTFS_TAR", "$OUT_ROOTFS_FLAT" and "$OUT_FSJSON" created.
|
||||
|
|
|
|||
|
|
@ -17,8 +17,6 @@ import tarfile
|
|||
|
||||
VERSION = 3
|
||||
|
||||
USE_COMPRESSION = True
|
||||
|
||||
IDX_NAME = 0
|
||||
IDX_SIZE = 1
|
||||
IDX_MTIME = 2
|
||||
|
|
@ -70,6 +68,8 @@ def main():
|
|||
args.add_argument("path",
|
||||
metavar="path-or-tar",
|
||||
help="Base path or tar file to include in JSON")
|
||||
args.add_argument("--zstd", action="store_true",
|
||||
help="Use Zstandard compression")
|
||||
|
||||
args = args.parse_args()
|
||||
|
||||
|
|
@ -81,9 +81,9 @@ def main():
|
|||
tar = None
|
||||
|
||||
if tar:
|
||||
(root, total_size) = handle_tar(logger, tar)
|
||||
(root, total_size) = handle_tar(logger, tar, args.zstd)
|
||||
else:
|
||||
(root, total_size) = handle_dir(logger, path, args.exclude)
|
||||
(root, total_size) = handle_dir(logger, path, args.exclude, args.zstd)
|
||||
|
||||
if False:
|
||||
# normalize the order of children, useful to debug differences between
|
||||
|
|
@ -105,7 +105,7 @@ def main():
|
|||
logger.info("Creating json ...")
|
||||
json.dump(result, args.out, check_circular=False, separators=(',', ':'))
|
||||
|
||||
def handle_dir(logger, path, exclude):
|
||||
def handle_dir(logger, path, exclude, use_compression):
|
||||
path = path + "/"
|
||||
exclude = exclude or []
|
||||
exclude = [os.path.join("/", os.path.normpath(p)) for p in exclude]
|
||||
|
|
@ -199,7 +199,7 @@ def handle_dir(logger, path, exclude):
|
|||
obj[IDX_TARGET] = target
|
||||
elif isfile:
|
||||
file_hash = hash_file(absname)
|
||||
filename = file_hash[0:HASH_LENGTH] + (".bin.zst" if USE_COMPRESSION else ".bin")
|
||||
filename = file_hash[0:HASH_LENGTH] + (".bin.zst" if use_compression else ".bin")
|
||||
existing = filename_to_hash.get(filename)
|
||||
assert existing is None or existing == file_hash, "Collision in short hash (%s and %s)" % (existing, file_hash)
|
||||
filename_to_hash[filename] = file_hash
|
||||
|
|
@ -214,7 +214,7 @@ def handle_dir(logger, path, exclude):
|
|||
|
||||
return (mainroot, total_size)
|
||||
|
||||
def handle_tar(logger, tar):
|
||||
def handle_tar(logger, tar, use_compression):
|
||||
mainroot = []
|
||||
filename_to_hash = {}
|
||||
total_size = 0
|
||||
|
|
@ -242,7 +242,7 @@ def handle_tar(logger, tar):
|
|||
obj[IDX_MODE] |= S_IFREG
|
||||
f = tar.extractfile(member)
|
||||
file_hash = hash_fileobj(f)
|
||||
filename = file_hash[0:HASH_LENGTH] + (".bin.zst" if USE_COMPRESSION else ".bin")
|
||||
filename = file_hash[0:HASH_LENGTH] + (".bin.zst" if use_compression else ".bin")
|
||||
existing = filename_to_hash.get(filename)
|
||||
assert existing is None or existing == file_hash, "Collision in short hash (%s and %s)" % (existing, file_hash)
|
||||
filename_to_hash[filename] = file_hash
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue