archive: preserve 4GiB+ entry sizes on LLP64

archive-tar.c

@@ -21,7 +21,7 @@
 #define BLOCKSIZE	(RECORDSIZE * 20)
 
 static char block[BLOCKSIZE];
-static unsigned long offset;
+static size_t offset;
 
 static int tar_umask = 002;
 
@@ -66,12 +66,12 @@ static void write_if_needed(void)
  * queues up writes, so that all our write(2) calls write exactly one
  * full block; pads writes to RECORDSIZE
  */
-static void do_write_blocked(const void *data, unsigned long size)
+static void do_write_blocked(const void *data, size_t size)
 {
 	const char *buf = data;
 
 	if (offset) {
-		unsigned long chunk = BLOCKSIZE - offset;
+		size_t chunk = BLOCKSIZE - offset;
 		if (size < chunk)
 			chunk = size;
 		memcpy(block + offset, buf, chunk);
@@ -93,7 +93,7 @@ static void do_write_blocked(const void *data, unsigned long size)
 
 static void finish_record(void)
 {
-	unsigned long tail;
+	size_t tail;
 	tail = offset % RECORDSIZE;
 	if (tail)  {
 		memset(block + offset, 0, RECORDSIZE - tail);
@@ -102,7 +102,7 @@ static void finish_record(void)
 	write_if_needed();
 }
 
-static void write_blocked(const void *data, unsigned long size)
+static void write_blocked(const void *data, size_t size)
 {
 	do_write_blocked(data, size);
 	finish_record();
@@ -114,7 +114,7 @@ static void write_blocked(const void *data, unsigned long size)
  */
 static void write_trailer(void)
 {
-	int tail = BLOCKSIZE - offset;
+	size_t tail = BLOCKSIZE - offset;
 	memset(block + offset, 0, tail);
 	write_block(block);
 	if (tail < 2 * RECORDSIZE) {
@@ -140,7 +140,7 @@ static int stream_blocked(struct repository *r, const struct object_id *oid)
 		readlen = odb_stream_read(st, buf, sizeof(buf));
 		if (readlen <= 0)
 			break;
-		do_write_blocked(buf, readlen);
+		do_write_blocked(buf, (size_t)readlen);
 	}
 	odb_stream_close(st);
 	if (!readlen)
@@ -218,10 +218,11 @@ static size_t get_path_prefix(const char *path, size_t pathlen, size_t maxlen)
 
 static void prepare_header(struct archiver_args *args,
 			   struct ustar_header *header,
-			   unsigned int mode, unsigned long size)
+			   unsigned int mode, size_t size)
 {
 	xsnprintf(header->mode, sizeof(header->mode), "%07o", mode & 07777);
-	xsnprintf(header->size, sizeof(header->size), "%011"PRIoMAX , S_ISREG(mode) ? (uintmax_t)size : (uintmax_t)0);
+	xsnprintf(header->size, sizeof(header->size), "%011"PRIoMAX,
+		  S_ISREG(mode) ? (uintmax_t)size : (uintmax_t)0);
 	xsnprintf(header->mtime, sizeof(header->mtime), "%011lo", (unsigned long) args->time);
 
 	xsnprintf(header->uid, sizeof(header->uid), "%07o", 0);
@@ -239,7 +240,7 @@ static void prepare_header(struct archiver_args *args,
 
 static void write_extended_header(struct archiver_args *args,
 				  const struct object_id *oid,
-				  const void *buffer, unsigned long size)
+				  const void *buffer, size_t size)
 {
 	struct ustar_header header;
 	unsigned int mode;
@@ -256,11 +257,11 @@ static int write_tar_entry(struct archiver_args *args,
 			   const struct object_id *oid,
 			   const char *path, size_t pathlen,
 			   unsigned int mode,
-			   void *buffer, unsigned long size)
+			   void *buffer, size_t size)
 {
 	struct ustar_header header;
 	struct strbuf ext_header = STRBUF_INIT;
-	unsigned long size_in_header;
+	size_t size_in_header;
 	int err = 0;
 
 	memset(&header, 0, sizeof(header));

archive-zip.c

@@ -201,9 +201,24 @@ static uint32_t clamp32(uintmax_t n)
 	return (n < max) ? n : max;
 }
 
-static void *zlib_deflate_raw(void *data, unsigned long size,
+static uint32_t crc32_size(uint32_t crc, const void *data, size_t size)
+{
+	const Bytef *buf = data;
+
+	while (size > 0) {
+		uInt chunk = size > UINT_MAX ? UINT_MAX : (uInt)size;
+
+		crc = crc32(crc, buf, chunk);
+		buf += chunk;
+		size -= chunk;
+	}
+
+	return crc;
+}
+
+static void *zlib_deflate_raw(void *data, size_t size,
 			      int compression_level,
-			      unsigned long *compressed_size)
+			      size_t *compressed_size)
 {
 	git_zstream stream;
 	size_t maxsize;
@@ -234,9 +249,9 @@ static void *zlib_deflate_raw(void *data, unsigned long size,
 	return buffer;
 }
 
-static void write_zip_data_desc(unsigned long size,
-				unsigned long compressed_size,
-				unsigned long crc)
+static void write_zip_data_desc(size_t size,
+				size_t compressed_size,
+				uint32_t crc)
 {
 	if (size >= 0xffffffff || compressed_size >= 0xffffffff) {
 		struct zip64_data_desc trailer;
@@ -250,21 +265,21 @@ static void write_zip_data_desc(unsigned long size,
 		struct zip_data_desc trailer;
 		copy_le32(trailer.magic, 0x08074b50);
 		copy_le32(trailer.crc32, crc);
-		copy_le32(trailer.compressed_size, compressed_size);
-		copy_le32(trailer.size, size);
+		copy_le32(trailer.compressed_size, (uint32_t)compressed_size);
+		copy_le32(trailer.size, (uint32_t)size);
 		write_or_die(1, &trailer, ZIP_DATA_DESC_SIZE);
 		zip_offset += ZIP_DATA_DESC_SIZE;
 	}
 }
 
 static void set_zip_header_data_desc(struct zip_local_header *header,
-				     unsigned long size,
-				     unsigned long compressed_size,
-				     unsigned long crc)
+				     size_t size,
+				     size_t compressed_size,
+				     uint32_t crc)
 {
 	copy_le32(header->crc32, crc);
-	copy_le32(header->compressed_size, compressed_size);
-	copy_le32(header->size, size);
+	copy_le32(header->compressed_size, clamp32(compressed_size));
+	copy_le32(header->size, clamp32(size));
 }
 
 static int has_only_ascii(const char *s)
@@ -295,7 +310,7 @@ static int write_zip_entry(struct archiver_args *args,
 			   const struct object_id *oid,
 			   const char *path, size_t pathlen,
 			   unsigned int mode,
-			   void *buffer, unsigned long size)
+			   void *buffer, size_t size)
 {
 	struct zip_local_header header;
 	uintmax_t offset = zip_offset;
@@ -304,8 +319,8 @@ static int write_zip_entry(struct archiver_args *args,
 	size_t header_extra_size = ZIP_EXTRA_MTIME_SIZE;
 	int need_zip64_extra = 0;
 	unsigned long attr2;
-	unsigned long compressed_size;
-	unsigned long crc;
+	size_t compressed_size;
+	uint32_t crc;
 	enum zip_method method;
 	unsigned char *out;
 	void *deflated = NULL;
@@ -318,7 +333,7 @@ static int write_zip_entry(struct archiver_args *args,
 	size_t zip_dir_extra_size = ZIP_EXTRA_MTIME_SIZE;
 	size_t zip64_dir_extra_payload_size = 0;
 
-	crc = crc32(0, NULL, 0);
+	crc = crc32_size(0, NULL, 0);
 
 	if (!has_only_ascii(path)) {
 		if (is_utf8(path))
@@ -355,7 +370,7 @@ static int write_zip_entry(struct archiver_args *args,
 			flags |= ZIP_STREAM;
 			out = NULL;
 		} else {
-			crc = crc32(crc, buffer, size);
+			crc = crc32_size(crc, buffer, size);
 			is_binary = entry_is_binary(args->repo->index,
 						    path_without_prefix,
 						    buffer, size);
@@ -431,12 +446,13 @@ static int write_zip_entry(struct archiver_args *args,
 			readlen = odb_stream_read(stream, buf, sizeof(buf));
 			if (readlen <= 0)
 				break;
-			crc = crc32(crc, buf, readlen);
+			crc = crc32_size(crc, buf, (size_t)readlen);
 			if (is_binary == -1)
 				is_binary = entry_is_binary(args->repo->index,
 							    path_without_prefix,
-							    buf, readlen);
-			write_or_die(1, buf, readlen);
+							    buf,
+							    (size_t)readlen);
+			write_or_die(1, buf, (size_t)readlen);
 		}
 		odb_stream_close(stream);
 		if (readlen)
@@ -464,14 +480,15 @@ static int write_zip_entry(struct archiver_args *args,
 			readlen = odb_stream_read(stream, buf, sizeof(buf));
 			if (readlen <= 0)
 				break;
-			crc = crc32(crc, buf, readlen);
+			crc = crc32_size(crc, buf, (size_t)readlen);
 			if (is_binary == -1)
 				is_binary = entry_is_binary(args->repo->index,
 							    path_without_prefix,
-							    buf, readlen);
+							    buf,
+							    (size_t)readlen);
 
 			zstream.next_in = buf;
-			zstream.avail_in = readlen;
+			zstream.avail_in = (size_t)readlen;
 			result = git_deflate(&zstream, 0);
 			if (result != Z_OK)
 				die(_("deflate error (%d)"), result);

archive.c

@@ -368,7 +368,8 @@ int write_archive_entries(struct archiver_args *args,
 			err = write_entry(args, &fake_oid,
 					  path, strlen(path),
 					  canon_mode(info->stat.st_mode),
-					  info->content, info->stat.st_size);
+					  info->content,
+					  (size_t)info->stat.st_size);
 		}
 
 		if (err)

archive.h

@@ -56,7 +56,7 @@ typedef int (*write_archive_entry_fn_t)(struct archiver_args *args,
 					const struct object_id *oid,
 					const char *path, size_t pathlen,
 					unsigned int mode,
-					void *buffer, unsigned long size);
+					void *buffer, size_t size);
 
 int write_archive_entries(struct archiver_args *args, write_archive_entry_fn_t write_entry);
 

t/meson.build

@@ -605,6 +605,7 @@ integration_tests = [
   't5002-archive-attr-pattern.sh',
   't5003-archive-zip.sh',
   't5004-archive-corner-cases.sh',
+  't5005-archive-large-files.sh',
   't5100-mailinfo.sh',
   't5150-request-pull.sh',
   't5200-update-server-info.sh',

t/t5005-archive-large-files.sh

@@ -0,0 +1,65 @@
+#!/bin/sh
+#
+# Copyright (c) 2026 Johannes Schindelin
+#
+
+test_description='git archive with 4GB+ entries'
+
+. ./test-lib.sh
+
+if ! test_have_prereq EXPENSIVE,SIZE_T_IS_64BIT
+then
+	skip_all='expensive 4GB archive test; enable on 64-bit with GIT_TEST_LONG=true'
+	test_done
+fi
+
+size_4gb=4294967296
+
+tar_info_size () {
+	"$TAR" tvf "$1" |
+	awk 'NR == 1 { print $3 }'
+}
+
+tar_gz_info_size () {
+	gzip -d -c <"$1" |
+	"$TAR" tvf - |
+	awk 'NR == 1 { print $3 }'
+}
+
+test_expect_success 'set up a 4GB file' '
+	test_atexit "rm -f large large.zip large.tar large.tar.gz" &&
+	# genrandom takes only an unsigned long...
+	test-tool genrandom 123 $(($size_4gb - 1)) >large &&
+	printf 1 >>large &&
+	test $size_4gb = $(test_file_size large)
+'
+
+test_expect_success 'add and commit 4GB file' '
+	git add large &&
+	git commit -m large-file
+'
+
+test_expect_success UNZIP 'zip archive stores 4GB file size' '
+	git archive -0 --format=zip HEAD >large.zip &&
+	"$GIT_UNZIP" -t large.zip &&
+	"$GIT_UNZIP" -l large.zip >large.zip.lst &&
+	awk '\''$NF == "large" { print $1; exit }'\'' <large.zip.lst >actual &&
+	echo $size_4gb >expect &&
+	test_cmp expect actual
+'
+
+test_expect_success 'tar archive stores 4GB file size' '
+	git archive --format=tar HEAD >large.tar &&
+	tar_info_size large.tar >actual &&
+	echo $size_4gb >expect &&
+	test_cmp expect actual
+'
+
+test_expect_success GZIP 'tar.gz archive stores 4GB file size' '
+	git archive --format=tar.gz HEAD >large.tar.gz &&
+	tar_gz_info_size large.tar.gz >actual &&
+	echo $size_4gb >expect &&
+	test_cmp expect actual
+'
+
+test_done