Merge pull request #527 from freifunk-gluon/modules

Generate opkg repo for kernel modules
This commit is contained in:
Matthias Schiffer 2015-10-30 14:15:04 +01:00
commit 99b7ed0080
19 changed files with 127 additions and 106 deletions

2
.gitignore vendored
View File

@ -1,6 +1,6 @@
*~
/build
/images
/output
/site
/openwrt
/packages

113
Makefile
View File

@ -59,29 +59,37 @@ CheckTarget := [ -n '$(GLUON_TARGET)' -a -n '$(GLUON_TARGET_$(GLUON_TARGET)_BOAR
CheckExternal := test -d $(GLUON_ORIGOPENWRTDIR) || (echo 'You don'"'"'t seem to have obtained the external repositories needed by Gluon; please call `make update` first!'; false)
create-key: FORCE
@$(CheckExternal)
+@$(GLUONMAKE_EARLY) create-key
prepare-target: FORCE
@$(CheckExternal)
@$(CheckTarget)
+@$(GLUONMAKE_EARLY) prepare-target
all: prepare-target
+@$(GLUONMAKE) build-key
+@$(GLUONMAKE) prepare
+@$(GLUONMAKE) images
+@$(GLUONMAKE) modules
prepare: prepare-target
+@$(GLUONMAKE) build-key
+@$(GLUONMAKE) $@
clean download images: FORCE
clean download images modules: FORCE
@$(CheckExternal)
@$(CheckTarget)
+@$(GLUONMAKE_EARLY) maybe-prepare-target
+@$(GLUONMAKE) build-key
+@$(GLUONMAKE) $@
toolchain/% package/% target/% image/%: FORCE
@$(CheckExternal)
@$(CheckTarget)
+@$(GLUONMAKE_EARLY) maybe-prepare-target
+@$(GLUONMAKE) build-key
+@$(GLUONMAKE) $@
manifest: FORCE
@ -103,13 +111,6 @@ manifest: FORCE
mkdir -p $(GLUON_IMAGEDIR)/sysupgrade
mv $(GLUON_BUILDDIR)/$(GLUON_BRANCH).manifest.tmp $(GLUON_IMAGEDIR)/sysupgrade/$(GLUON_BRANCH).manifest
update-vermagic: FORCE
@$(CheckExternal)
+($(foreach GLUON_TARGET,$(GLUON_TARGETS), \
$(GLUONMAKE_EARLY) maybe-prepare-target GLUON_TARGET='$(GLUON_TARGET)' V=s$(OPENWRT_VERBOSE) && \
$(GLUONMAKE) update-vermagic GLUON_TARGET='$(GLUON_TARGET)' V=s$(OPENWRT_VERBOSE) && \
) :)
dirclean : FORCE
for dir in build_dir dl staging_dir tmp; do \
rm -rf $(GLUON_ORIGOPENWRTDIR)/$$dir; \
@ -176,19 +177,6 @@ GLUON_$(1)_MODEL_$(2)_ALIASES += $(3)
endef
include $(GLUONDIR)/targets/targets.mk
include $(GLUONDIR)/targets/$(GLUON_TARGET)/profiles.mk
BOARD := $(GLUON_TARGET_$(GLUON_TARGET)_BOARD)
override SUBTARGET := $(GLUON_TARGET_$(GLUON_TARGET)_SUBTARGET)
target_prepared_stamp := $(BOARD_BUILDDIR)/target-prepared
gluon_prepared_stamp := $(BOARD_BUILDDIR)/prepared
include $(INCLUDE_DIR)/target.mk
prereq: FORCE
+$(NO_TRACE_MAKE) prereq
@ -216,7 +204,43 @@ feeds: FORCE
gluon-tools: FORCE
+$(GLUONMAKE_EARLY) tools/sed/install
+$(GLUONMAKE_EARLY) package/lua/host/install
+$(GLUONMAKE_EARLY) package/lua/host/install package/usign/host/install
prepare-early: FORCE
for dir in build_dir dl staging_dir; do \
mkdir -p $(GLUON_ORIGOPENWRTDIR)/$$dir; \
done
+$(GLUONMAKE_EARLY) feeds
+$(GLUONMAKE_EARLY) gluon-tools
create-key: prepare-early
[ -s $(GLUON_OPKG_KEY) -a -s $(GLUON_OPKG_KEY).pub ] || \
$(STAGING_DIR_HOST)/bin/usign -G -s $(GLUON_OPKG_KEY) -p $(GLUON_OPKG_KEY).pub -c "Gluon opkg key"
include $(GLUONDIR)/targets/targets.mk
ifneq ($(GLUON_TARGET),)
include $(GLUONDIR)/targets/$(GLUON_TARGET)/profiles.mk
BOARD := $(GLUON_TARGET_$(GLUON_TARGET)_BOARD)
override SUBTARGET := $(GLUON_TARGET_$(GLUON_TARGET)_SUBTARGET)
target_prepared_stamp := $(BOARD_BUILDDIR)/target-prepared
gluon_prepared_stamp := $(BOARD_BUILDDIR)/prepared
PREPARED_RELEASE = $$(cat $(gluon_prepared_stamp))
IMAGE_PREFIX = gluon-$(GLUON_SITE_CODE)-$(PREPARED_RELEASE)
MODULE_PREFIX = gluon-$(GLUON_SITE_CODE)-$(PREPARED_RELEASE)
include $(INCLUDE_DIR)/target.mk
build-key: FORCE
ln -sf $(GLUON_OPKG_KEY) $(BUILD_KEY)
ln -sf $(GLUON_OPKG_KEY).pub $(BUILD_KEY).pub
config: FORCE
+$(NO_TRACE_MAKE) scripts/config/conf OPENWRT_BUILD= QUIET=0
@ -235,31 +259,19 @@ config: FORCE
| sed -e 's/ /\n/g'; \
) > $(BOARD_BUILDDIR)/config.tmp
scripts/config/conf --defconfig=$(BOARD_BUILDDIR)/config.tmp Config.in
mv .config $(BOARD_BUILDDIR)/config
echo 'CONFIG_ALL_KMODS=y' >> $(BOARD_BUILDDIR)/config.tmp
scripts/config/conf --defconfig=$(BOARD_BUILDDIR)/config.tmp Config.in
mv .config $(BOARD_BUILDDIR)/config-allmods
cp $(BOARD_BUILDDIR)/config .config
prepare-target: FORCE
prepare-target: create-key
rm $(GLUON_OPENWRTDIR)/tmp || true
mkdir -p $(GLUON_OPENWRTDIR)/tmp
for dir in build_dir dl staging_dir; do \
mkdir -p $(GLUON_ORIGOPENWRTDIR)/$$dir; \
done
for link in build_dir config Config.in dl include Makefile package rules.mk scripts staging_dir target toolchain tools; do \
ln -sf $(GLUON_ORIGOPENWRTDIR)/$$link $(GLUON_OPENWRTDIR); \
done
+$(GLUONMAKE_EARLY) feeds
+$(GLUONMAKE_EARLY) gluon-tools
+$(GLUONMAKE) config
touch $(target_prepared_stamp)
$(target_prepared_stamp):
$(target_prepared_stamp): create-key
+$(GLUONMAKE_EARLY) prepare-target
maybe-prepare-target: $(target_prepared_stamp)
@ -291,8 +303,8 @@ toolchain: $(toolchain/stamp-install) $(tools/stamp-install)
include $(INCLUDE_DIR)/kernel.mk
kernel: FORCE
+$(NO_TRACE_MAKE) -C $(TOPDIR)/target/linux/$(BOARD) -f $(GLUONDIR)/include/Makefile.target $(LINUX_DIR)/.image TARGET_BUILD=1
+$(NO_TRACE_MAKE) -C $(TOPDIR)/target/linux/$(BOARD) -f $(GLUONDIR)/include/Makefile.target $(LINUX_DIR)/.modules TARGET_BUILD=1
+$(NO_TRACE_MAKE) -C $(TOPDIR)/target/linux/$(BOARD) $(LINUX_DIR)/.image TARGET_BUILD=1
+$(NO_TRACE_MAKE) -C $(TOPDIR)/target/linux/$(BOARD) $(LINUX_DIR)/.modules TARGET_BUILD=1
packages: $(package/stamp-compile)
$(_SINGLE)$(SUBMAKE) -r package/index
@ -320,6 +332,14 @@ prepare: FORCE
$(gluon_prepared_stamp):
+$(GLUONMAKE) prepare
modules: FORCE $(gluon_prepared_stamp)
-rm -f $(GLUON_MODULEDIR)/*/$(BOARD)/$(if $(SUBTARGET),$(SUBTARGET),generic)/*
-rmdir -p $(GLUON_MODULEDIR)/*/$(BOARD)/$(if $(SUBTARGET),$(SUBTARGET),generic)
mkdir -p $(GLUON_MODULEDIR)/$(MODULE_PREFIX)/$(BOARD)/$(if $(SUBTARGET),$(SUBTARGET),generic)
cp $(PACKAGE_DIR)/kmod-*.ipk $(GLUON_MODULEDIR)/$(MODULE_PREFIX)/$(BOARD)/$(if $(SUBTARGET),$(SUBTARGET),generic)
$(_SINGLE)$(SUBMAKE) -r package/index PACKAGE_DIR=$(GLUON_MODULEDIR)/$(MODULE_PREFIX)/$(BOARD)/$(if $(SUBTARGET),$(SUBTARGET),generic)
include $(INCLUDE_DIR)/package-ipkg.mk
@ -332,9 +352,6 @@ BIN_DIR = $(PROFILE_BUILDDIR)/images
TARGET_DIR = $(PROFILE_BUILDDIR)/root
PREPARED_RELEASE = $$(cat $(gluon_prepared_stamp))
IMAGE_PREFIX = gluon-$(GLUON_SITE_CODE)-$(PREPARED_RELEASE)
OPKG:= \
IPKG_TMP="$(TMP_DIR)/ipkgtmp" \
IPKG_INSTROOT="$(TARGET_DIR)" \
@ -447,15 +464,7 @@ manifest: FORCE
) : \
) >> $(GLUON_BUILDDIR)/$(GLUON_BRANCH).manifest.tmp
update-vermagic: FORCE
mkdir -p '$(BOARD_BUILDDIR)'
echo '$(DEFAULT_OPKG_REPO)' > '$(BOARD_BUILDDIR)/default_opkg_repo'
$(VERSION_SED) '$(BOARD_BUILDDIR)/default_opkg_repo'
wget -q -O- "$$(cat '$(BOARD_BUILDDIR)/default_opkg_repo')/base/Packages.gz" \
| gzip -d \
| awk '/Depends: kernel / { match($$3,/[[:xdigit:]]{32}/,m); print m[0]; exit }' \
> $(GLUONDIR)/targets/$(GLUON_TARGET)/vermagic
.PHONY: all images prepare clean gluon-tools manifest update-vermagic
.PHONY: all create-key prepare images modules clean gluon-tools manifest
endif
endif

View File

@ -76,21 +76,9 @@ A directory for the new target must be created under ``targets``, and it must be
to ``targets/targets.mk``. In the new target directory, the following files must be created:
* profiles.mk
* vermagic
* config (optional)
For ``profiles.mk``, see :ref:`hardware-adding-profiles`.
The file ``config`` can be used to add additional, target-specific options to the OpenWrt config.
The files ``vermagic`` must have the correct content so kernel modules from the upstream repositories
can be installed without dependency issues. The OpenWrt version a Gluon release is based on is defined by the upstream package repo URL in ``include/gluon.mk``
(in the variable ``DEFAULT_OPKG_REPO``); at the time this documentation was written, this was ``barrier_breaker/14.07``; whenever
the package repo is updated, all ``vermagic`` files must be updated as well. The command ``make update-vermagic`` can be used to get the correct
vermagic, or update the values when ``DEFAULT_OPKG_REPO`` has changed.
The content is a hash which is part of the version number of the kernel package. So in the case of ``ar71xx-generic`` on
``barrier_breaker``, we look for the kernel package in the directory ``https://downloads.openwrt.org/barrier_breaker/14.07/ar71xx/generic/packages/base/``.
As the kernel package is called ``kernel_3.10.49-1-0114c71ed85677c9c1e4911437af4743_ar71xx.ipk``, the correct ``vermagic`` string
is ``0114c71ed85677c9c1e4911437af4743``.
After this, is should be sufficient to call ``make GLUON_TARGET=<target>`` to build the images for the new target.

View File

@ -30,10 +30,10 @@ A fully automated nightly build could use the following commands:
make clean
make -j5 GLUON_TARGET=ar71xx-generic GLUON_BRANCH=experimental
make manifest GLUON_BRANCH=experimental
contrib/sign.sh $SECRETKEY images/sysupgrade/experimental.manifest
contrib/sign.sh $SECRETKEY output/images/sysupgrade/experimental.manifest
rm -rf /where/to/put/this/experimental
cp -r images /where/to/put/this/experimental
cp -r output/images /where/to/put/this/experimental
Infrastructure

View File

@ -86,7 +86,7 @@ In case of errors read the messages carefully and try to fix the stated issues (
``ar71xx-generic`` is the most common target and will generated images for most of the supported hardware.
To see a complete list of supported targets, call ``make`` without setting ``GLUON_TARGET``.
The built images can be found in the directory `images`. Of these, the factory
The built images can be found in the directory `output/images`. Of these, the factory
images are to be used when flashing from the original firmware a device came with,
and sysupgrade is to upgrade from other versions of Gluon or any other OpenWRT-based
system.
@ -109,19 +109,63 @@ will clean the entire tree, so the toolchain will be rebuilt as well, which is
not necessary in most cases, and will take a while.
opkg repositories
-----------------
Gluon is mostly compatible with OpenWrt, so the normal OpenWrt package repositories
can be used for Gluon as well. It is advisable to setup a mirror or reverse proxy
reachable over IPv6 and add it to ``site.conf`` as http://downloads.openwrt.org/ does
not support IPv6.
This is not true for kernel modules; the Gluon kernel is incompatible with the
kernel of the default OpenWrt images. Therefore, Gluon will not only generate images,
but also an opkg repositoy containing all kernel modules provided by OpenWrt/Gluon
for the kernel of the generated images.
Signing keys
............
Gluon does not support HTTPS for downloading packages; fortunately, opkg deploys
public-key cryptography to ensure package integrity.
The Gluon images will contain two public keys: the official OpenWrt signing key
(to allow installing userspace packages) and a Gluon-specific key (which is used
to sign the generated module repository).
By default, Gluon will handle the generation and handling of the keys itself.
When making firmware releases based on Gluon, it might make sense to store the
the keypair, so updating the module repository later is possible.
The location the keys are stored at and read from can be changed
(see :ref:`getting-started-environment-variables`). To only generate the keypair
at the configured location without doing a full build, use ``make create-key``.
.. _getting-started-environment-variables:
Environment variables
---------------------
Gluon's build process can be controlled by various environment variables.
GLUON_SITEDIR
Path to the site configuration. Defaults to ``site/``.
GLUON_IMAGEDIR
Path where images will be stored. Defaults to ``images/``.
Path to the site configuration. Defaults to ``site``.
GLUON_BUILDDIR
Working directory during build. Defaults to ``build/``.
Working directory during build. Defaults to ``build``.
GLUON_OPKG_KEY
Path key file used to sign the module opkg repository. Defaults to ``$(GLUON_BULDDIR)/gluon-opkg-key``.
The private key will be stored as ``$(GLUON_OPKG_KEY)``, the public key as ``$(GLUON_OPKG_KEY).pub``.
GLUON_OUTPUTDIR
Path where output files will be stored. Defaults to ``output``.
GLUON_IMAGEDIR
Path where images will be stored. Defaults to ``$(GLUON_OUTPUTDIR)/images``.
GLUON_MODULEDIR
Path where the kernel module opkg repository will be stored. Defaults to ``$(GLUON_OUTPUTDIR)/modules``.
So all in all, to update and rebuild a Gluon build tree, the following commands should be used (repeat the

View File

@ -1,19 +0,0 @@
# code adjusted from openwrt/include/kernel-defaults.mk
override define Kernel/Configure
$(LINUX_CONF_CMD) > $(LINUX_DIR)/.config.target
# copy CONFIG_KERNEL_* settings over to .config.target
awk '/^(#[[:space:]]+)?CONFIG_KERNEL/{sub("CONFIG_KERNEL_","CONFIG_");print}' $(BOARD_BUILDDIR)/config-allmods >> $(LINUX_DIR)/.config.target
echo "# CONFIG_KALLSYMS_EXTRA_PASS is not set" >> $(LINUX_DIR)/.config.target
echo "# CONFIG_KALLSYMS_ALL is not set" >> $(LINUX_DIR)/.config.target
echo "# CONFIG_KALLSYMS_UNCOMPRESSED is not set" >> $(LINUX_DIR)/.config.target
$(SCRIPT_DIR)/metadata.pl kconfig $(TMP_DIR)/.packageinfo $(BOARD_BUILDDIR)/config-allmods $(KERNEL_PATCHVER) > $(LINUX_DIR)/.config.override
$(SCRIPT_DIR)/kconfig.pl 'm+' '+' $(LINUX_DIR)/.config.target /dev/null $(LINUX_DIR)/.config.override > $(LINUX_DIR)/.config
$(call Kernel/SetNoInitramfs)
rm -rf $(KERNEL_BUILD_DIR)/modules
$(_SINGLE) [ -d $(LINUX_DIR)/user_headers ] || $(MAKE) $(KERNEL_MAKEOPTS) INSTALL_HDR_PATH=$(LINUX_DIR)/user_headers headers_install
cp $(GLUONDIR)/targets/$(GLUON_TARGET)/vermagic $(LINUX_DIR)/.vermagic
endef
# The Makefile included here is $(TOPDIR)/target/linux/$(BOARD)/Makefile
include Makefile

View File

@ -1,7 +1,7 @@
CONFIG_IMAGEOPT=y
# CONFIG_PER_FEED_REPO is not set
CONFIG_DEVEL=y
CONFIG_ALL_KMODS=y
CONFIG_BUSYBOX_CUSTOM=y
CONFIG_BUSYBOX_CONFIG_SHA512SUM=y

View File

@ -2,13 +2,18 @@ ifneq ($(__gluon_inc),1)
__gluon_inc=1
GLUON_SITEDIR ?= $(GLUONDIR)/site
GLUON_IMAGEDIR ?= $(GLUONDIR)/images
GLUON_BUILDDIR ?= $(GLUONDIR)/build
GLUON_ORIGOPENWRTDIR := $(GLUONDIR)/openwrt
GLUON_SITE_CONFIG := $(GLUON_SITEDIR)/site.conf
export GLUONDIR GLUON_SITEDIR GLUON_SITE_CONFIG GLUON_IMAGEDIR GLUON_BUILDDIR
GLUON_OUTPUTDIR ?= $(GLUONDIR)/output
GLUON_IMAGEDIR ?= $(GLUON_OUTPUTDIR)/images
GLUON_MODULEDIR ?= $(GLUON_OUTPUTDIR)/modules
GLUON_OPKG_KEY ?= $(GLUON_BUILDDIR)/gluon-opkg-key
export GLUONDIR GLUON_SITEDIR GLUON_BUILDDIR GLUON_SITE_CONFIG GLUON_OUTPUTDIR GLUON_IMAGEDIR GLUON_MODULEDIR
BOARD_BUILDDIR = $(GLUON_BUILDDIR)/$(GLUON_TARGET)
@ -16,6 +21,11 @@ BOARD_KDIR = $(BOARD_BUILDDIR)/kernel
export BOARD_BUILDDIR
LINUX_RELEASE := 2
export LINUX_RELEASE
GLUON_OPENWRTDIR = $(BOARD_BUILDDIR)/openwrt

View File

@ -1 +0,0 @@
7bed08fa9c06eb8089e82c200340ec66

View File

@ -1 +0,0 @@
a44d8090b8f752ec1d96871dbbb8988a

View File

@ -1 +0,0 @@
57d4b7d13c23edd1a689dd6d6adc65ad

View File

@ -1 +0,0 @@
0fec8065ae15bf9934ffbe2512e8e39f

View File

@ -1 +0,0 @@
521812ea85032e662b3d3469a3a10df8

View File

@ -1 +0,0 @@
c8b57a131072a3198e594822481af3e0

View File

@ -1 +0,0 @@
3c199b535784c56bd7a6b4d6ad82b91f

View File

@ -1 +0,0 @@
30da46d39f906146155850351fa0acd9

View File

@ -1 +0,0 @@
8549f8163c15d79b053f26aa0d52e96f

View File

@ -1 +0,0 @@
d0d2aab265f2535385cf8d756315d27c

View File

@ -1 +0,0 @@
bce85557e89d5a4f30d6d723503b1a61