build: backport kernel debuginfo ZSTD compression
This commit is contained in:
Matthias Schiffer
parent
ed094bc68c
commit
e5456bdf14
|
|
@ -0,0 +1,113 @@
|
|||
From: Matthias Schiffer <mschiffer@universe-factory.net>
|
||||
Date: Wed, 13 May 2020 20:22:12 +0200
|
||||
Subject: tools: add zstd
|
||||
|
||||
Signed-off-by: Matthias Schiffer <mschiffer@universe-factory.net>
|
||||
(cherry picked from commit 258dc0d0fd3aae47add9b7dca40848a92d03a4ea)
|
||||
|
||||
diff --git a/tools/Makefile b/tools/Makefile
|
||||
index d7207ba89dd91df558eaf970961fdef225aa1f37..14fe4fb4b5f4e0c745cb8592a39bcf238dcc5444 100644
|
||||
--- a/tools/Makefile
|
||||
+++ b/tools/Makefile
|
||||
@@ -33,7 +33,7 @@ tools-$(CONFIG_TARGET_mxs) += elftosb sdimage
|
||||
tools-$(CONFIG_TARGET_ar71xx) += lzma-old
|
||||
tools-$(CONFIG_TARGET_ar71xx)$(CONFIG_TARGET_ath79) += squashfs
|
||||
tools-$(CONFIG_USES_MINOR) += kernel2minor
|
||||
-tools-y += lzma squashfskit4 zip
|
||||
+tools-y += lzma squashfskit4 zip zstd
|
||||
tools-$(BUILD_B43_TOOLS) += b43-tools
|
||||
tools-$(BUILD_ISL) += isl
|
||||
tools-$(CONFIG_USE_SPARSE) += sparse
|
||||
diff --git a/tools/zstd/Makefile b/tools/zstd/Makefile
|
||||
new file mode 100644
|
||||
index 0000000000000000000000000000000000000000..7459725e8e79b846ed96551753d07fdd02459598
|
||||
--- /dev/null
|
||||
+++ b/tools/zstd/Makefile
|
||||
@@ -0,0 +1,20 @@
|
||||
+include $(TOPDIR)/rules.mk
|
||||
+
|
||||
+PKG_NAME:=zstd
|
||||
+PKG_VERSION:=1.4.4
|
||||
+
|
||||
+PKG_SOURCE:=$(PKG_NAME)-$(PKG_VERSION).tar.gz
|
||||
+PKG_SOURCE_URL:=@GITHUB/facebook/zstd/releases/download/v$(PKG_VERSION)
|
||||
+PKG_HASH:=a364f5162c7d1a455cc915e8e3cf5f4bd8b75d09bc0f53965b0c9ca1383c52c8
|
||||
+
|
||||
+PKG_LICENSE:=BSD-3-Clause
|
||||
+PKG_LICENSE_FILES:=LICENSE
|
||||
+PKG_CPE_ID:=cpe:/a:facebook:zstandard
|
||||
+
|
||||
+HOST_BUILD_PARALLEL:=1
|
||||
+
|
||||
+include $(INCLUDE_DIR)/host-build.mk
|
||||
+
|
||||
+HOST_MAKE_FLAGS = PREFIX=$(HOST_BUILD_PREFIX) HAVE_ZLIB=0 HAVE_LZMA=0 HAVE_LZ4=0
|
||||
+
|
||||
+$(eval $(call HostBuild))
|
||||
diff --git a/tools/zstd/patches/0001-build-issue-More-portable-header-prefix-usage-1987.patch b/tools/zstd/patches/0001-build-issue-More-portable-header-prefix-usage-1987.patch
|
||||
new file mode 100644
|
||||
index 0000000000000000000000000000000000000000..6d743aa3855262c2a0956f70ec99588ce9ebe53b
|
||||
--- /dev/null
|
||||
+++ b/tools/zstd/patches/0001-build-issue-More-portable-header-prefix-usage-1987.patch
|
||||
@@ -0,0 +1,61 @@
|
||||
+From 06a57cf57e3c4e887cadcf688e3081154f3f6db4 Mon Sep 17 00:00:00 2001
|
||||
+Message-Id: <06a57cf57e3c4e887cadcf688e3081154f3f6db4.1589392463.git.mschiffer@universe-factory.net>
|
||||
+From: Bimba Shrestha <bimbashrestha@fb.com>
|
||||
+Date: Thu, 6 Feb 2020 14:10:51 -0800
|
||||
+Subject: [PATCH] [build-issue] More portable header prefix usage (#) (#1987)
|
||||
+
|
||||
+* make 4.3 build issue fix
|
||||
+
|
||||
+* Changing header name and adding comment
|
||||
+---
|
||||
+ programs/Makefile | 11 +++++++----
|
||||
+ 1 file changed, 7 insertions(+), 4 deletions(-)
|
||||
+
|
||||
+diff --git a/programs/Makefile b/programs/Makefile
|
||||
+index b75314a83f43..a9ee3cb5311b 100644
|
||||
+--- a/programs/Makefile
|
||||
++++ b/programs/Makefile
|
||||
+@@ -94,9 +94,12 @@ endif
|
||||
+
|
||||
+ VOID = /dev/null
|
||||
+
|
||||
++# Make 4.3 doesn't support '\#' anymore (https://lwn.net/Articles/810071/)
|
||||
++NUM_SYMBOL := \#
|
||||
++
|
||||
+ # thread detection
|
||||
+ NO_THREAD_MSG := ==> no threads, building without multithreading support
|
||||
+-HAVE_PTHREAD := $(shell printf '\#include <pthread.h>\nint main(void) { return 0; }' > have_pthread.c && $(CC) $(FLAGS) -o have_pthread$(EXT) have_pthread.c -pthread 2> $(VOID) && rm have_pthread$(EXT) && echo 1 || echo 0; rm have_pthread.c)
|
||||
++HAVE_PTHREAD := $(shell printf '$(NUM_SYMBOL)include <pthread.h>\nint main(void) { return 0; }' > have_pthread.c && $(CC) $(FLAGS) -o have_pthread$(EXT) have_pthread.c -pthread 2> $(VOID) && rm have_pthread$(EXT) && echo 1 || echo 0; rm have_pthread.c)
|
||||
+ HAVE_THREAD := $(shell [ "$(HAVE_PTHREAD)" -eq "1" -o -n "$(filter Windows%,$(OS))" ] && echo 1 || echo 0)
|
||||
+ ifeq ($(HAVE_THREAD), 1)
|
||||
+ THREAD_MSG := ==> building with threading support
|
||||
+@@ -108,7 +111,7 @@ endif
|
||||
+
|
||||
+ # zlib detection
|
||||
+ NO_ZLIB_MSG := ==> no zlib, building zstd without .gz support
|
||||
+-HAVE_ZLIB := $(shell printf '\#include <zlib.h>\nint main(void) { return 0; }' > have_zlib.c && $(CC) $(FLAGS) -o have_zlib$(EXT) have_zlib.c -lz 2> $(VOID) && rm have_zlib$(EXT) && echo 1 || echo 0; rm have_zlib.c)
|
||||
++HAVE_ZLIB := $(shell printf '$(NUM_SYMBOL)include <zlib.h>\nint main(void) { return 0; }' > have_zlib.c && $(CC) $(FLAGS) -o have_zlib$(EXT) have_zlib.c -lz 2> $(VOID) && rm have_zlib$(EXT) && echo 1 || echo 0; rm have_zlib.c)
|
||||
+ ifeq ($(HAVE_ZLIB), 1)
|
||||
+ ZLIB_MSG := ==> building zstd with .gz compression support
|
||||
+ ZLIBCPP = -DZSTD_GZCOMPRESS -DZSTD_GZDECOMPRESS
|
||||
+@@ -119,7 +122,7 @@ endif
|
||||
+
|
||||
+ # lzma detection
|
||||
+ NO_LZMA_MSG := ==> no liblzma, building zstd without .xz/.lzma support
|
||||
+-HAVE_LZMA := $(shell printf '\#include <lzma.h>\nint main(void) { return 0; }' > have_lzma.c && $(CC) $(FLAGS) -o have_lzma$(EXT) have_lzma.c -llzma 2> $(VOID) && rm have_lzma$(EXT) && echo 1 || echo 0; rm have_lzma.c)
|
||||
++HAVE_LZMA := $(shell printf '$(NUM_SYMBOL)include <lzma.h>\nint main(void) { return 0; }' > have_lzma.c && $(CC) $(FLAGS) -o have_lzma$(EXT) have_lzma.c -llzma 2> $(VOID) && rm have_lzma$(EXT) && echo 1 || echo 0; rm have_lzma.c)
|
||||
+ ifeq ($(HAVE_LZMA), 1)
|
||||
+ LZMA_MSG := ==> building zstd with .xz/.lzma compression support
|
||||
+ LZMACPP = -DZSTD_LZMACOMPRESS -DZSTD_LZMADECOMPRESS
|
||||
+@@ -130,7 +133,7 @@ endif
|
||||
+
|
||||
+ # lz4 detection
|
||||
+ NO_LZ4_MSG := ==> no liblz4, building zstd without .lz4 support
|
||||
+-HAVE_LZ4 := $(shell printf '\#include <lz4frame.h>\n\#include <lz4.h>\nint main(void) { return 0; }' > have_lz4.c && $(CC) $(FLAGS) -o have_lz4$(EXT) have_lz4.c -llz4 2> $(VOID) && rm have_lz4$(EXT) && echo 1 || echo 0; rm have_lz4.c)
|
||||
++HAVE_LZ4 := $(shell printf '$(NUM_SYMBOL)include <lz4frame.h>\n\#include <lz4.h>\nint main(void) { return 0; }' > have_lz4.c && $(CC) $(FLAGS) -o have_lz4$(EXT) have_lz4.c -llz4 2> $(VOID) && rm have_lz4$(EXT) && echo 1 || echo 0; rm have_lz4.c)
|
||||
+ ifeq ($(HAVE_LZ4), 1)
|
||||
+ LZ4_MSG := ==> building zstd with .lz4 compression support
|
||||
+ LZ4CPP = -DZSTD_LZ4COMPRESS -DZSTD_LZ4DECOMPRESS
|
||||
+--
|
||||
+2.26.2
|
||||
+
|
||||
|
|
@ -0,0 +1,43 @@
|
|||
From: Matthias Schiffer <mschiffer@universe-factory.net>
|
||||
Date: Wed, 13 May 2020 20:33:46 +0200
|
||||
Subject: build: compress kernel debuginfo using zstd
|
||||
|
||||
zstd with its default settings (compression level -3) compresses better
|
||||
than bzip2 -9 (which is the default setting), and is an order of magnitude
|
||||
faster.
|
||||
|
||||
I made the following measurements for the most common compression tools
|
||||
(all standard Debian Buster versions, default flags unless noted
|
||||
otherwise), using the debug information of a large x86-64 kernel with
|
||||
ALL_KMODS:
|
||||
|
||||
* kernel-debug.tar: 376M
|
||||
* kernel-debug.tar.gz: 101M, compressed in ~12s
|
||||
* kernel-debug.tar.bz2: 91M, compressed in ~15s
|
||||
* kernel-debug.tar.xz: 57M, compressed in ~101s
|
||||
* kernel-debug.tar.zst: 86M, compressed in ~1s
|
||||
|
||||
With zstd, there is still some room for improvement by increasing the
|
||||
compression, but the slight increase in compression ratio
|
||||
(22.83% -> 19.46%) does not justify the significant increase in
|
||||
compression time (about 5 times on my machine) in my opinion.
|
||||
|
||||
Note that multithreaded compression (-T argument) does not affect
|
||||
reproducibility with zstd.
|
||||
|
||||
Signed-off-by: Matthias Schiffer <mschiffer@universe-factory.net>
|
||||
(cherry picked from commit 4bd7990488b0ca7b5cae16f0a9147a4146759053)
|
||||
|
||||
diff --git a/include/kernel-build.mk b/include/kernel-build.mk
|
||||
index 3fdf7efc52857a5184348cc1261848f75751b8a9..af7c3a8f0bb15c7e0d7072876705ff0bf4f9c8d1 100644
|
||||
--- a/include/kernel-build.mk
|
||||
+++ b/include/kernel-build.mk
|
||||
@@ -70,7 +70,7 @@ ifdef CONFIG_COLLECT_KERNEL_DEBUG
|
||||
$(FIND) $(KERNEL_BUILD_DIR)/debug -type f | $(XARGS) $(KERNEL_CROSS)strip --only-keep-debug
|
||||
$(TAR) c -C $(KERNEL_BUILD_DIR) debug \
|
||||
$(if $(SOURCE_DATE_EPOCH),--mtime="@$(SOURCE_DATE_EPOCH)") \
|
||||
- | bzip2 -c -9 > $(BIN_DIR)/kernel-debug.tar.bz2
|
||||
+ | zstd -T0 -f -o $(BIN_DIR)/kernel-debug.tar.zst
|
||||
endef
|
||||
endif
|
||||
|
||||
Loading…
Reference in New Issue