From 1ba8f8ad675a26131f994c5552bc6d00ca80ab41 Mon Sep 17 00:00:00 2001 From: Rasmus Villemoes Date: Mon, 7 Nov 2016 13:11:41 +0100 Subject: [PATCH 01/10] recipes: begin section on common variables This introduces the beginning of the definition of a custom oe "domain" in sphinx-lingo, which I think should allow us to define special markup for e.g. variables etc. --- .gitignore | 1 + source/conf.py | 7 ++- source/examples/jansson_2.9.oe.sig | 1 + source/handbook/__init__.py | 0 source/handbook/oedomain.py | 32 +++++++++++ source/recipes.rst | 89 +++++++++++++++++++++++++++++- 6 files changed, 124 insertions(+), 6 deletions(-) create mode 100644 source/examples/jansson_2.9.oe.sig create mode 100644 source/handbook/__init__.py create mode 100644 source/handbook/oedomain.py diff --git a/.gitignore b/.gitignore index c9213b7..93c5d40 100644 --- a/.gitignore +++ b/.gitignore @@ -2,3 +2,4 @@ *.html *.pdf build/ +*.pyc diff --git a/source/conf.py b/source/conf.py index 43fd873..2d02d98 100644 --- a/source/conf.py +++ b/source/conf.py @@ -16,9 +16,9 @@ # add these directories to sys.path here. If the directory is relative to the # documentation root, use os.path.abspath to make it absolute, like shown here. # -# import os -# import sys -# sys.path.insert(0, os.path.abspath('.')) +import os +import sys +sys.path.insert(0, os.path.abspath('.')) # -- General configuration ------------------------------------------------ @@ -32,6 +32,7 @@ extensions = [ 'sphinx.ext.todo', 'sphinx.ext.ifconfig', + 'handbook.oedomain', ] # Add any paths that contain templates here, relative to this directory. diff --git a/source/examples/jansson_2.9.oe.sig b/source/examples/jansson_2.9.oe.sig new file mode 100644 index 0000000..fe05110 --- /dev/null +++ b/source/examples/jansson_2.9.oe.sig @@ -0,0 +1 @@ +d8c04a344c37afff17cd17ffb3ca2a8a08c1eafc jansson-2.9.tar.bz2 diff --git a/source/handbook/__init__.py b/source/handbook/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/source/handbook/oedomain.py b/source/handbook/oedomain.py new file mode 100644 index 0000000..6cf7cca --- /dev/null +++ b/source/handbook/oedomain.py @@ -0,0 +1,32 @@ +# -*- coding: utf-8 -*- +""" + oedomain + ~~~~~~~~~~ + + An OE-lite domain. + + This domain initially provides an `oe:var` role. + + Based on GNU Make domain from + +""" + +__version__ = "0.1.1" +# for this module's sphinx doc +release = __version__ +version = release.rsplit('.', 1)[0] + +from sphinxcontrib.domaintools import custom_domain +import re + +def setup(app): + app.add_domain(custom_domain('OEliteDomain', + name = 'oe', + label = "OE-lite", + + elements = dict( + var = dict( + objname = "OE-lite Variable", + indextemplate = "pair: %s; OE-lite Variable" + ), + ))) diff --git a/source/recipes.rst b/source/recipes.rst index 8baa0d3..b3cc3d2 100644 --- a/source/recipes.rst +++ b/source/recipes.rst @@ -13,7 +13,7 @@ conventions used as well as a short example. When starting a build (``oe bake``), OE-lite starts by parsing all recipe files in all registered layers, pruning recipes which are not -compatible with the current target architecture. +compatible with the current target architecture. Naming conventions @@ -70,7 +70,7 @@ files. A complete recipe file can be as small as: .. code-block:: oe :caption: meta/base/recipes/vim/vim_7.4.oe - + require ${PN}.inc The ``require`` directive instructs OE-lite to look for the given file @@ -88,7 +88,7 @@ of an ordinary assignment ``FOO = "BAR"`` is not expanded until ``FOO`` is expanded, whereas the ``:=`` operator causes immediate expansion of the RHS. Also, the operator ``+=`` appends the RHS value to the LHS variable, but also prepends a space if the variable was -non-empty. +non-empty. Note, however, that OE-lite does not have the concept of variable »flavors«, and that all right-hand sides should be properly quoted @@ -97,3 +97,86 @@ strings. See the appendix `chap_syntax` for a semi-formal survey of the various allowed syntactic elements. +Variables +========= + +This section describes some of the more common variables one needs to +define or which are just nice to know about. + +Source code +----------- + +.. oe:var:: SRC_URI + + A space-separated list of stuff to fetch to build the recipe. Each + element should be a valid URI using one of the recognized + schemes: + + - http\:// + - ftp\:// + - file\:// + - git\:// + + The http and ftp schemes usually refer to (possibly compressed) + tarballs. OE-lite recognizes these and automatically handles + decompressing/unpacking in the do_unpack task. + + The file scheme is for local files included with the recipe, + which may for example be a configuration file which should end up + getting copied to /etc, or a patch which needs to be applied before + building. + + The git scheme can be used to refer to a (local or remote) git + repository. This scheme also accepts a number of parameters. These + are given as key=value pairs separated from each other and the main + uri by semicolons. The possible parameters are: + + - ``protocol`` the protocol which git will use when cloning from + the given uri. The default is ``git``, but other possibilities + are ``http`` and ``ssh``. Using the ``ssh`` protocol typically + requires that public key authentification has been set up (that + is, the user running OE-lite has a public key which allows + password-less login to the remote server). + + - ``commit`` a sha1 to check out. + + - ``tag`` a tag to check out. + + - ``branch`` a branch to check out. + + The latter three are mutually exclusive. + + It is quite common for the :oe:var:`SRC_URI` variable to be defined + in terms of the recipe version, ``${PV}``. A few examples:: + + SRC_URI = "http://www.digip.org/jansson/releases/jansson-${PV}.tar.bz2" + SRC_URI = "git://git.sv.gnu.org/libunwind.git;tag=v${PV}" + SRC_URI = "git://github.com/kergoth/tslib.git;commit=f6c499a523bff845ddd57b1d96c9d1389f0df17b" + SRC_URI = "${GNU_MIRROR}/autoconf/autoconf-${PV}.tar.bz2" + +Checksums +~~~~~~~~~ + +To check the *integrity* of the downloaded files, OE-lite computes a +SHA1 checksum and compares it to value in recipe's *signature file* - +a file with the same name as the recipe file and ``.sig`` +appended. For example: + +.. literalinclude:: examples/jansson_2.9.oe.sig + :caption: recipes/jansson/jansson_2.9.oe.sig + +If the signature file does not exist, OE-lite creates one +automatically during do_fetch, but then deliberately causes the build +to fail to alert the user. This can be a useful way to create the +signature file for a new ingredient. The format of the signature files +is the same as the output from the standard ``sha1sum`` utility, so +another way is to download the file manually and run + +.. code-block:: sh + + sha1sum jansson-2.9.tar.bz2 > /path/to/recipedir/jansson_2.9.oe.sig + +However the signature file is generated, it is up to the creator of +the recipe to ensure its *authenticity*, e.g. by comparing a checksum +of the downloaded file to one provided by the upstream project (which +is not necessarily a SHA1). From 3332d3635c657f72afa2695179157ec922ad0e4c Mon Sep 17 00:00:00 2001 From: Rasmus Villemoes Date: Wed, 9 Nov 2016 15:33:08 +0100 Subject: [PATCH 02/10] recipes.rst: add section on dependencies Understanding dependencies and how to define them is rather crucial, so don't hide this away deep inside an example. --- source/recipe_examples.rst | 91 +++++++++----------------------------- source/recipes.rst | 83 ++++++++++++++++++++++++++++++++++ 2 files changed, 104 insertions(+), 70 deletions(-) diff --git a/source/recipe_examples.rst b/source/recipe_examples.rst index 9d09a26..b0fda17 100644 --- a/source/recipe_examples.rst +++ b/source/recipe_examples.rst @@ -22,13 +22,13 @@ magic. So we create this directory tree:: │   └── trig.c └── trig_0.1.oe - + The file ``trig.c`` is simply: - + .. literalinclude:: examples/trig/trig.c :language: c :caption: recipes/trig/files/trig.c - + Our first attempt at describing how to build this to OE-lite is this: .. literalinclude:: examples/trig/trig_0.1.oe @@ -91,7 +91,7 @@ Let's try this: waiting for machine:trig_0.1:do_compile (started 0.020 seconds ago) to finish ERROR: machine:trig_0.1:do_compile failed - 0.023 s Build: 0.674 seconds - + ERROR: machine:trig_0.1:do_compile failed /mnt/xfs/devel/oe-lite/tmp/work/machine/arm-926ejs-linux-gnueabi/trig-0.1/tmp/do_compile.20161102082713.log > LC_ALL=C /mnt/xfs/devel/oe-lite/tmp/work/machine/arm-926ejs-linux-gnueabi/trig-0.1/tmp/do_compile.20161102082713.run + cd /mnt/xfs/devel/oe-lite/tmp/work/machine/arm-926ejs-linux-gnueabi/trig-0.1/src/trig-0.1 @@ -102,7 +102,7 @@ Let's try this: arm-926ejs-linux-gnueabi-gcc: fatal error: no input files compilation terminated. Error: Command failed: 'LC_ALL=C /mnt/xfs/devel/oe-lite/tmp/work/machine/arm-926ejs-linux-gnueabi/trig-0.1/tmp/do_compile.20161102082713.run': 1 - + CRITICAL: bake failed: error: 1 It can sometimes be difficult to see what the problem actually @@ -146,7 +146,7 @@ what does *there* mean? Let's inspect the workdir: ├── do_stage.log -> do_stage.20161102082713.log ├── do_unpack.20161102082713.log └── do_unpack.log -> do_unpack.20161102082713.log - + 21 directories, 10 files Here we see the problem: do_compile was run in the @@ -161,7 +161,7 @@ section `task-directories`), but ``trig.c`` has been put in With this in place, let's try again. .. code-block:: console - + oe bake trig -y ... + cd /mnt/xfs/devel/oe-lite/tmp/work/machine/arm-926ejs-linux-gnueabi/trig-0.1/src @@ -172,7 +172,7 @@ With this in place, let's try again. trig.c:(.text.startup+0x2c): undefined reference to `sin' collect2: error: ld returned 1 exit status Error: Command failed: 'LC_ALL=C /mnt/xfs/devel/oe-lite/tmp/work/machine/arm-926ejs-linux-gnueabi/trig-0.1/tmp/do_compile.20161102091655.run': 1 - + CRITICAL: bake failed: error: 1 Right, we didn't provide the ``-lm`` linker flag. OK, that's easy to fix. @@ -183,7 +183,7 @@ Right, we didn't provide the ``-lm`` linker flag. OK, that's easy to fix. Once more. What can possibly go wrong now? .. code-block:: console - + $ oe bake trig -y ... ERROR: machine:trig_0.1:do_compile failed /mnt/xfs/devel/oe-lite/tmp/work/machine/arm-926ejs-linux-gnueabi/trig-0.1/tmp/do_compile.20161102092251.log @@ -195,7 +195,7 @@ Once more. What can possibly go wrong now? /mnt/xfs/devel/oe-lite/tmp/work/machine/arm-926ejs-linux-gnueabi/trig-0.1/stage/cross/bin/../lib/gcc/arm-926ejs-linux-gnueabi/5.4.0/../../../../arm-926ejs-linux-gnueabi/bin/ld: cannot find -lm collect2: error: ld returned 1 exit status Error: Command failed: 'LC_ALL=C /mnt/xfs/devel/oe-lite/tmp/work/machine/arm-926ejs-linux-gnueabi/trig-0.1/tmp/do_compile.20161102092251.run': 1 - + CRITICAL: bake failed: error: 1 That this fails is actually a good thing, because it shows that @@ -220,62 +220,14 @@ libm gets included, but that's not necessarily the case for other libraries, so it's better to always explicitly describe the exact dependencies.) -Aside: dependency types ------------------------ - -So why did we spell the runtime dependency ``RDEPENDS_${PN}`` and not -just ``RDEPENDS``? There are actually two kinds of build-time as well -as two types of run-time dependencies, *recipe dependencies* and -*package dependencies*. Recipe dependencies (given in the unsuffixed -``DEPENDS``, ``RDEPENDS`` variables) describe what is required to -build the recipe. Package dependencies, given in ``DEPENDS_``, ``RDEPENDS_`` somewhere, but -``bar.h`` contains a ``#include ``. That libbar depends on -libfoo is an implementation detail of libbar, which frob doesn't care -about (and it may change with a different version of libbar), but in -this case we obviously need to ensure that ``foo.h`` gets staged when -building frob. The solution to this is to ensure that the package -providing libbar has a build-time dependency on libfoo. So the libbar -recipe might contain :: - - DEPENDS += "libfoo" - DEPENDS_${PN} += "libfoo-dev" - RDEPENDS_${PN} += "libfoo" - -which says that (1) libfoo is necessary to build libbar, (2) to build -anything against libbar, you also need the libfoo-dev package, (3) if -you run-time depend on libbar, you also run-time depend on libfoo. - -The alert reader may wonder how a *run-time dependency* for *building* -a recipe makes any sense. And in truth, most normal recipes do not -have those – a bare ``RDEPENDS`` in a recipe is usually an -error. However, there is one type of recipes which do have -``RDEPENDS``: Those that inherit image.oeclass, and hence describe a -complete file system image. While normal recipes have a do_stage task, -which pulls in all packages mentioned in the recipe's ``DEPENDS`` -variable as well as their package dependencies (recursively), image -recipes have an do_rstage task which pulls in all the packages in the -recipe's ``RDEPENDS`` variable as well as their package rdependencies -(recursively). It is admittedly a stretch to call this run-time build -dependencies, but as the preceding sentence hopefully demonstrates, -this makes the handling of the two staging tasks nicely symmetric. - -Back to the example -------------------- +Since our utilities end up in the package by the same name as the +recipe, we tell OE-lite that anything that run-time depends on the +``trig`` *package* should also pull in libm. + +See the section `dependencies` for details on the various types of +dependencies and the variables describing them, and why the runtime +dependency was spelled ``RDEPENDS_${PN}`` rather than just +``RDEPENDS`` While we can now succesfully build the trig utilities, the recipe is not quite complete. Looking at the directory ``${WORKDIR}/packages``, @@ -310,7 +262,7 @@ and puts them in the ``.debug`` subdirectory: │   └── tan* ├── sin* └── tan* - + 3 directories, 6 files The next task is do_split, which takes the contents of the ``${D}`` @@ -338,7 +290,7 @@ have reasonable default values, so we get this structure: ├── trig-dev/ ├── trig-doc/ └── trig-locale/ - + 10 directories, 6 files This allows one to RDEPEND on ``trig``, but if one also wants the @@ -383,7 +335,6 @@ might want to improve. Instead of showing how to achieve this, we'll turn our attention to an example from »real life«. - + Dissection of an existing recipe ================================ - diff --git a/source/recipes.rst b/source/recipes.rst index b3cc3d2..ee70263 100644 --- a/source/recipes.rst +++ b/source/recipes.rst @@ -180,3 +180,86 @@ However the signature file is generated, it is up to the creator of the recipe to ensure its *authenticity*, e.g. by comparing a checksum of the downloaded file to one provided by the upstream project (which is not necessarily a SHA1). + +.. _dependencies: + +Dependencies +------------ + +There are a few different types of dependencies one needs to know +about – and these dependencies must be described to OE-lite using +these variables: + +.. oe:var:: DEPENDS + + Describes recipe build dependencies – items that are necessary to + build the recipe. + +.. oe:var:: RDEPENDS + + Describes recipe runtime dependencies – not used in ordinary + recipes (see below). + +.. oe:var:: DEPENDS_foo + + Describes package build dependencies – additional items that are + necessary to build something that depends on the foo package. + +.. oe:var:: RDEPENDS_foo + + Describes package runtime dependencies – additional items that are + necessary at runtime for using something that has a runtime + dependency on the foo package. + +OE-lite distinguishes between *build* and *runtime* +dependencies. A *build* dependency is something that is necessary to +build a given recipe, while a *runtime* dependency describes some +other item (typically a shared library) that is needed to actually run +the software. + +These are further divided into *recipe* dependencies and *package* +dependencies. Recipe dependencies (given in the unsuffixed +``DEPENDS``, ``RDEPENDS`` variables) describe what is required to +build the recipe. Package dependencies, given in ``DEPENDS_``, ``RDEPENDS_``, describe what is needed to use +the contents of the package at build-time respectively run-time. + +An example where package build-time dependencies would come into play +is if we have two libraries, libfoo and libbar and a utility frob, +with libfoo depending on libbar and frob depending on libbar. In the +frob recipe, we would then have something like:: + + DEPENDS += "libbar" + RDEPENDS_${PN} += "libbar" + +The frob utility probably does a ``#include `` somewhere, but +``bar.h`` contains a ``#include ``. That libbar depends on +libfoo is an implementation detail of libbar, which frob doesn't care +about (and it may change with a different version of libbar), but in +this case we obviously need to ensure that ``foo.h`` gets staged when +building frob. The solution to this is to ensure that the package +providing libbar has a build-time dependency on libfoo. So the libbar +recipe might contain :: + + DEPENDS += "libfoo" + DEPENDS_${PN} += "libfoo-dev" + RDEPENDS_${PN} += "libfoo" + +which says that (1) libfoo is necessary to build libbar, (2) to build +anything against libbar, you also need the libfoo-dev package, (3) if +you run-time depend on libbar, you also run-time depend on libfoo. + +The alert reader may wonder how a *run-time dependency* for *building* +a recipe makes any sense. And in truth, most normal recipes do not +have those – a bare ``RDEPENDS`` in a recipe is usually an +error. However, there is one type of recipes which do have +``RDEPENDS``: Those that inherit image.oeclass, and hence describe a +complete file system image. While normal recipes have a do_stage task, +which pulls in all packages mentioned in the recipe's ``DEPENDS`` +variable as well as their package *build* dependencies (recursively), +image recipes have a do_rstage task which pulls in all the packages in +the recipe's ``RDEPENDS`` variable as well as their package *runtime* +dependencies (recursively). It is admittedly a stretch to call this +run-time build dependencies, but as the preceding sentence hopefully +demonstrates, this makes the handling of the two staging tasks nicely +symmetric. From cad3d6cc2dcaa3949c5609eb70c3ef2add1fec01 Mon Sep 17 00:00:00 2001 From: Rasmus Villemoes Date: Thu, 10 Nov 2016 16:48:11 +0100 Subject: [PATCH 03/10] recipes.rst: begin section on USE flags --- source/recipes.rst | 85 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 85 insertions(+) diff --git a/source/recipes.rst b/source/recipes.rst index ee70263..4676c66 100644 --- a/source/recipes.rst +++ b/source/recipes.rst @@ -263,3 +263,88 @@ dependencies (recursively). It is admittedly a stretch to call this run-time build dependencies, but as the preceding sentence hopefully demonstrates, this makes the handling of the two staging tasks nicely symmetric. + + +USE flags +--------- + +To make it possible to tweak various aspects of a recipe without +having to modify the recipe itself, OE-lite has the concept of *USE +flags*. These are special variables, called ``USE_*``, which are not +set directly but rather receive their value from settings in various +configuration files. The value of ``USE_foo`` is the value of one of +these variables: + +- ``DEFAULT_USE_foo`` (lowest priority) +- ``DISTRO_USE_foo`` (intended for distro configs) +- ``MACHINE_USE_foo`` (intended for machine configs) +- ``LOCAL_USE_foo`` (intended for local.conf) +- ``RECIPE_USE_foo`` (highest priority) + +The idea is that one can, for example, have a generic setting +``DISTRO_USE_foo = "1"`` in a ``distro.conf`` file, while overriding +that with ``MACHINE_USE_foo = "0"`` in a machine-specific +configuration file. + +.. oe:var:: RECIPE_FLAGS + + This is a space-separated list of USE flag names (without the + ``USE_`` prefix) that apply to the recipe. Only those USE flags + listed in :oe:var:`RECIPE_FLAGS` are available as variables. + +Since USE flags are set globally, it is good practice to have the +first word of the name be the same as the recipe name. + +A USE flag example +~~~~~~~~~~~~~~~~~~ + +An example from the `freetype` recipe:: + + RECIPE_FLAGS += "freetype_bzip2" + EXTRA_OECONF += "${EXTRA_OECONF_BZIP2}" + EXTRA_OECONF_BZIP2 = "--without-bzip2" + EXTRA_OECONF_BZIP2:USE_freetype_bzip2 = "--with-bzip2" + DEPENDS:>USE_freetype_bzip2 = " libbz2" + DEPENDS_${PN}:>USE_freetype_bzip2 = " libbz2" + RDEPENDS_${PN}:>USE_freetype_bzip2 = " libbz2" + +This defines a USE flag ``USE_freetype_bzip2`` which is used to decide +whether freetype should be compiled with support for bz2-compressed +fonts. + +The logic in the three ``EXTRA_OECONF`` lines is that we append the +value of the ``EXTRA_OECONF_BZIP2`` variables to the ``EXTRA_OECONF`` +variable - as explained below in the `autotools` section, the contents +of that variable is appended to the ``./configure`` command line when +a recipe uses the ``autotools`` class. The next line defines the +``EXTRA_OECONF_BZIP2`` variable with the contents ``--without-bzip2``, +while the third line uses the `override` mechanism to set its value to +``--with-bzip2``. Altogether, this ensures that freetype gets +configured appropriately. + +But the recipe author's job is not quite done yet. When the USE flag +is set, building the recipe also requires libbz2, so we need +to, conditionally, append libbz2 to the ``DEPENDS`` variable. + +The freetype library ends up in the package also called freetype, so +we declare libbz2 as a build as well as runtime dependency of the +freetype package. + +USE flag gotchas +~~~~~~~~~~~~~~~~ + +Most USE flags are boolean flags, so they are usually just assigned a +value of False or True, but they can really be any string. When used +in an override assignment, the strings ``""`` and ``"0"`` as well as +the value False are treated as false (that is, the override is not +applied), while all others are treated as true. + +Due to a quirk of the current implementation, USE flag variables whose +final value is either the empty string or the string ``"0"`` end up +not being defined at all. This can unfortunately manifest itself as +rather obscure and hard-to-debug errors such as ``Exception: cannot +concatenate 'str' and 'NoneType' objects``, or worse not be detected +at build-time at all. It is also worth noting that using the tokens +``True`` and ``False`` on the right-hand side of assignments is +treated exactly as if one used ``"1"`` and ``"0"``, respectively. + From abd701e193067308645f794f36ccb81ae7162871 Mon Sep 17 00:00:00 2001 From: Rasmus Villemoes Date: Tue, 20 Dec 2016 12:57:32 +0100 Subject: [PATCH 04/10] source/recipes.rst: add sections on autotools and make classes --- source/recipes.rst | 67 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 67 insertions(+) diff --git a/source/recipes.rst b/source/recipes.rst index 4676c66..06ed4e2 100644 --- a/source/recipes.rst +++ b/source/recipes.rst @@ -348,3 +348,70 @@ at build-time at all. It is also worth noting that using the tokens ``True`` and ``False`` on the right-hand side of assignments is treated exactly as if one used ``"1"`` and ``"0"``, respectively. +.. _autotools_class: + +autotools +--------- + +The autotools `class ` is useful for software that uses +the `GNU Build System +`_, usually just known +as Autotools. It provides suitable implementations of +``do_configure``, ``do_compile`` and ``do_install`` (the latter two +being inherited from the `make_class` class) For example, +``do_configure`` invokes the autoconf ``configure`` script with all +the cross-compile relevant arguments (``--build``, ``--host``, +``--prefix``, ``cross_compiling=yes`` etc. etc.) set appropriately. + +.. oe:var:: EXTRA_OECONF + +One can augment the ``./configure`` commandline by setting the +:oe:var:`EXTRA_OECONF` variable – the value of this variable is simply +appended to the ``./configure`` invocation. We saw an example of this +in the USE flag example above, where ``EXTRA_OECONF`` was made to +contain either ``--with-bzip2`` or ``--without-bzip2`` depending on +the value of the ``USE_freetype_bzip2`` flag. + +Since each piece of software has a different set of ``--enable-*``, +``--disable-*``, ``--with-*``, ``--without-*`` etc. flags, it is up to +the recipe author to add any appropriate options to +``EXTRA_OECONF``. Making each feature controllable with a USE flag can +be quite cumbersome, and there's nothing wrong with starting out with +just unconditionally adding e.g. ``--without-udev``. + +.. _make_class: + +make +---- + +The make `class ` is useful for software that is built +and installed using standard ``make`` and ``make install`` +commands. It provides suitable definitions of ``do_compile`` and +``do_install`` that, essentially, run ``make`` and ``make install``, +respectively. + +.. oe:var:: EXTRA_OEMAKE +.. oe:var:: PARALLEL_MAKE +.. oe:var:: EXTRA_OEMAKE_COMPILE +.. oe:var:: EXTRA_OEMAKE_INSTALL +.. oe:var:: MAKE_DESTDIR + +The contents of the :oe:var:`EXTRA_OEMAKE` variable is appended to the +``make`` command line in both cases. In addition, +:oe:var:`PARALLEL_MAKE` and :oe:var:`EXTRA_OEMAKE_COMPILE` are appended +in the ``do_compile`` case, while :oe:var:`EXTRA_OEMAKE_INSTALL` and +:oe:var:`MAKE_DESTDIR` are appended in the ``do_install`` case. + +The ``EXTRA_OEMAKE*`` variables are for recipe-specific tweaks; often +they are not needed at all. + +:oe:var:`PARALLEL_MAKE` is typically defined in ``local.conf`` and +hence applies to all recipes, containing something like ``-j8`` or +however many cpu cores one has (or wishes to use). Some software is +known not to support parallel builds – in those cases, one can set +``PARALLEL_MAKE = ""`` or more explicitly ``PARALLEL_MAKE = "-j1"`` in +the recipe file to force a serial build. + +:oe:var:`MAKE_DESTDIR` usually contains the value ``DESTDIR=${D}``, and +thus serves to ensure that the ``DESTDIR`` variable is defined +appropriately. From 9c2c182745f26ad8372f7a59c57fcb7e9e247100 Mon Sep 17 00:00:00 2001 From: Rasmus Villemoes Date: Tue, 20 Dec 2016 12:58:19 +0100 Subject: [PATCH 05/10] source/recipes.rst: fixup reference --- source/recipes.rst | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/source/recipes.rst b/source/recipes.rst index 06ed4e2..b92a5dd 100644 --- a/source/recipes.rst +++ b/source/recipes.rst @@ -314,13 +314,13 @@ fonts. The logic in the three ``EXTRA_OECONF`` lines is that we append the value of the ``EXTRA_OECONF_BZIP2`` variables to the ``EXTRA_OECONF`` -variable - as explained below in the `autotools` section, the contents -of that variable is appended to the ``./configure`` command line when -a recipe uses the ``autotools`` class. The next line defines the -``EXTRA_OECONF_BZIP2`` variable with the contents ``--without-bzip2``, -while the third line uses the `override` mechanism to set its value to -``--with-bzip2``. Altogether, this ensures that freetype gets -configured appropriately. +variable - as explained below in the `autotools_class` section, the +contents of that variable is appended to the ``./configure`` command +line when a recipe uses the ``autotools`` class. The next line defines +the ``EXTRA_OECONF_BZIP2`` variable with the contents +``--without-bzip2``, while the third line uses the `override` +mechanism to set its value to ``--with-bzip2``. Altogether, this +ensures that freetype gets configured appropriately. But the recipe author's job is not quite done yet. When the USE flag is set, building the recipe also requires libbz2, so we need From 7690346bb6f4ce774fb68b442166c9cf754a568b Mon Sep 17 00:00:00 2001 From: Rasmus Villemoes Date: Tue, 20 Dec 2016 16:28:35 +0100 Subject: [PATCH 06/10] source/tasks.rst: expand do_split section Also add some link targets and remove an incomplete sentence (no idea what I wanted to say) from do_install. --- source/tasks.rst | 25 +++++++++++++++++++++---- 1 file changed, 21 insertions(+), 4 deletions(-) diff --git a/source/tasks.rst b/source/tasks.rst index 770cc98..04fe026 100644 --- a/source/tasks.rst +++ b/source/tasks.rst @@ -200,22 +200,26 @@ do_compile ~~~~~~~~~~ This task is where the software actually gets built. In many cases -this is just calling ``make``. The working directory is ``${S}``. +this is just calling ``make``. Task dependencies: configure Working directory: ``${B}`` +.. _do_install_task: + do_install ~~~~~~~~~~ -This installs the software under ``${WORKDIR}/install``, often just by -invoking ``make install``. During +This installs the software under ``${D}`` aka ``${WORKDIR}/install``, +often just by invoking ``make install``. Task dependencies: compile Working directory: ``${B}`` +.. _do_split_task: + do_split ~~~~~~~~ @@ -224,7 +228,20 @@ packages. Files belonging to the package ``foo`` gets copied to a directory tree under ``${PKGD}/foo``. The splitting is governed by the ``FILES_*`` variables. These contain space-separated lists of glob patterns. For example, ``FILES_${PN}-dev`` contain (among other -things) ``/lib/lib*.so /usr/include``, so all +things) ``/lib/lib*.so /usr/include``, so all files matching +``/lib/lib*.so`` as well as all files under ``/usr/include`` belong to +the ``FILES_${PN}-dev`` package. + +Each file under ``${D}`` (that is, the files generated by the +`do_install_task` task) belongs to one and only one package. Packages +are processed in the order they appear in the :oe:var:`PACKAGES` +variable, and the first package with a `FILES_*` glob pattern matching +a given file gets it. This means that when creating custom packages, +one usually uses the prepend operator ``=+`` to add the new package +name. + +If, after processing all packages, there are files not claimed by any +package, the do_split task fails. Task dependencies: install From 051f7acc8d49b3f72178cbf8cb2bd901cfdadef7 Mon Sep 17 00:00:00 2001 From: Rasmus Villemoes Date: Tue, 20 Dec 2016 16:29:50 +0100 Subject: [PATCH 07/10] source/recipes.rst: document auto-package-libs.oeclass --- source/recipes.rst | 76 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 76 insertions(+) diff --git a/source/recipes.rst b/source/recipes.rst index b92a5dd..b0898a2 100644 --- a/source/recipes.rst +++ b/source/recipes.rst @@ -415,3 +415,79 @@ the recipe file to force a serial build. :oe:var:`MAKE_DESTDIR` usually contains the value ``DESTDIR=${D}``, and thus serves to ensure that the ``DESTDIR`` variable is defined appropriately. + +.. _auto_package_libs_class: + +auto-package-libs +----------------- + +Some recipes provide multiple (more or less related) libraries. The +auto-package-libs `class ` provides a convenient way to +split those libraries into separate `packages `. It +is easiest to look at an example to to explain how it is used. + +The mosquitto recipe generates two libraries libmosquitto and +libmosquittopp, the latter being a C++ version. In ``mosquitto.inc`` +we find:: + + inherit auto-package-libs + AUTO_PACKAGE_LIBS = "mosquitto mosquittopp" + +That is, the :oe:var:`AUTO_PACKAGE_LIBS` variable should contain the +list of library names, each without the ``lib`` prefix. For each +library name X, this automatically creates the packages +mosquitto-libX, mosquitto-libX-dev and mosquitto-libX-dbg [»creating« +a package is really just a matter of adding a word to the +:oe:var:`PACKAGES` variable], and also creates corresponding +``FILES_*`` variables naming the files which belong to those +packages. For example, for the three mosquittopp packages, the +``FILES_*`` variables contain:: + + FILES_mosquitto-libmosquittopp='/usr/lib/libmosquittopp.so.*' + FILES_mosquitto-libmosquittopp-dbg='/usr/lib/.debug/libmosquittopp.so.*' + FILES_mosquitto-libmosquittopp-dev='/usr/lib/libmosquittopp.so /usr/lib/libmosquittopp.la /usr/lib/libmosquittopp.a /usr/lib/pkgconfig/mosquittopp.pc' + +The `do_split_task` task puts every file matching those glob patterns +into the corresponding package. + +Unfortunately, the two lines above are not quite sufficient to explain +everything to OE-lite - in particular, we need to define the `package +dependencies ` for the individual packages, so that a +build or run-time dependency on e.g. libmosquittopp automatically +pulls in anything else that that library needs. For convenience, four +variables are available that can be set to common dependencies for all +the libX and libX-dev packages: + +.. oe:var:: AUTO_PACKAGE_LIBS_DEPENDS + + automatically added to each package build dependency DEPENDS_foo-libX. + +.. oe:var:: AUTO_PACKAGE_LIBS_RDEPENDS + + automatically added to each package run-time dependency variable RDEPENDS_foo-libX. + +.. oe:var:: AUTO_PACKAGE_LIBS_DEV_DEPENDS + + automatically added to each package build dependency DEPENDS_foo-libX-dev. + +.. oe:var:: AUTO_PACKAGE_LIBS_DEV_RDEPENDS + + automatically added to each package run-time dependency variable RDEPENDS_foo-libX-dev. + +For example, the mosquitto.inc file contains:: + + AUTO_PACKAGE_LIBS_DEPENDS = "libpthread librt libz libm openssl" + AUTO_PACKAGE_LIBS_RDEPENDS = "libc libcrypto libpthread librt libssl" + +In addition, the C++ library depends on the C library, so we additionally have:: + + DEPENDS_mosquitto-libmosquittopp += "libmosquitto libstdc++" + RDEPENDS_mosquitto-libmosquittopp += "libmosquitto libgcc libstdc++" + +For hopefully obvious reasons, libmosquitto cannot be added to the +common dependency variable AUTO_PACKAGE_LIBS_DEPENDS, but on the other +hand it is crucial that depending on libmosquittopp in some other +recipe also implies a dependency on libmosquitto. One could add +libstdc++ to the common dependency variable, but that would +unnecessarily pollute the staging area for recipes that only depend on +the C library. From 7e5e932227ba11166f44832f3ed1ebd3419148ea Mon Sep 17 00:00:00 2001 From: Rasmus Villemoes Date: Thu, 22 Dec 2016 13:28:00 +0100 Subject: [PATCH 08/10] recipes.rst: document auto-package-utils class --- source/handbook/oedomain.py | 6 +++ source/recipes.rst | 97 +++++++++++++++++++++++++++++++++++++ 2 files changed, 103 insertions(+) diff --git a/source/handbook/oedomain.py b/source/handbook/oedomain.py index 6cf7cca..c8c58a3 100644 --- a/source/handbook/oedomain.py +++ b/source/handbook/oedomain.py @@ -29,4 +29,10 @@ def setup(app): objname = "OE-lite Variable", indextemplate = "pair: %s; OE-lite Variable" ), + # abbreviated because we can't use the Python reserved + # word "class" as a keyword argument, sigh + cls = dict( + objname = "OE-lite Class", + indextemplate = "pair: %s; OE-lite Class" + ), ))) diff --git a/source/recipes.rst b/source/recipes.rst index b0898a2..2b7194c 100644 --- a/source/recipes.rst +++ b/source/recipes.rst @@ -491,3 +491,100 @@ recipe also implies a dependency on libmosquitto. One could add libstdc++ to the common dependency variable, but that would unnecessarily pollute the staging area for recipes that only depend on the C library. + +.. _auto_package_utils_class: + +auto-package-utils +------------------ + +A companion to the `auto_package_libs_class` class, the +auto-package-utils class is useful for splitting utilities into +separate packages. In principle, the usage is as simple as:: + + inherit auto-packages-utils + AUTO_PACKAGE_UTILS = "bzip2 bzdiff bzgrep bzip2recover bzmore" + +This example is from the bzip2 recipe. The above creates packages +bzip2-X and bzip2-X-doc for each X in the ``AUTO_PACKAGE_UTILS`` +variable. The corresponding ``FILES_*`` automatically get sensible +values, e.g. :: + + FILES_bzip2-bzdiff = "/sbin/bzdiff /bin/bzdiff /usr/sbin/bzdiff /usr/bin/bzdiff /usr/libexec/bzdiff" + FILES_bzip2-bzdiff-doc = "/usr/share/man/man?/bzdiff.*" + +ensures that the bzdiff binary, whereever it might get installed, ends +up in the bzip2-bzdiff package, and its documentation goes into +bzip2-bzdiff-doc. + +There are also corresponding ``PROVIDES_*`` settings:: + + PROVIDES_bzip2-bzdiff = "util/bzdiff" + +The ``util/`` prefix is a convention used to designate »utilities« +(anything executable) for dependency purposes. By default, a package +provides an item with the same name as the package, so if one writes :: + + RDEPENDS += "bzip2" + +one would get whatever got packaged in the main ``bzip2`` package, +which doesn't necessarily (and in fact, not in this case) include the +``bzip2`` binary. Hence if one needs the ``bzip2`` utility, one should +spell it :: + + RDEPENDS += "util/bzip2" + +That will make OE-lite find and stage the package which actually +provides the ``bzip2`` binary, and other recipes do not need to know +about how the bzip2 recipe is being split. + +Some utilities behave differently depending on how they are invoked +(i.e., based on ``argv[0]``). This is typically implemented by making +each of the alternate names a symlink to the main binary. Since these +symlinks would be useless by themselves, it is better to package them +together with the binary they point to. The +:oe:cls:`auto-package-utils` class provides a simple mechanism for +this: Set ``AUTO_PACKAGE_UTILS_SYMLINKS_foo`` to the list for +alternate names for the ``foo`` utility. For example, in the bzip2 +recipe, we have:: + + AUTO_PACKAGE_UTILS_SYMLINKS_bzip2 = "bunzip2 bzcat" + AUTO_PACKAGE_UTILS_SYMLINKS_bzdiff = "bzcmp" + AUTO_PACKAGE_UTILS_SYMLINKS_bzgrep = "bzegrep bzfgrep" + AUTO_PACKAGE_UTILS_SYMLINKS_bzmore = "bzless" + +and the above ``FILES_*`` and ``PROVIDES_*`` were slightly lying; +their real values are:: + + FILES_bzip2-bzdiff = "/sbin/bzdiff /bin/bzdiff /usr/sbin/bzdiff /usr/bin/bzdiff /usr/libexec/bzdiff /sbin/bzcmp /bin/bzcmp /usr/sbin/bzcmp /usr/bin/bzcmp /usr/libexec/bzcmp" + FILES_bzip2-bzdiff-doc = "/usr/share/man/man?/bzdiff.* /usr/share/man/man?/bzcmp.*" + PROVIDES_bzip2-bzdiff = "util/bzdiff util/bzcmp" + +Just as for auto-package-libs, one also needs to define the package +dependencies for the packages created by the auto-package-utils class, +and there are similar convenience variables for defining common +dependencies: + +.. oe:var:: AUTO_PACKAGE_UTILS_DEPENDS + + automatically added to each package build dependency DEPENDS_foo-X. + +.. oe:var:: AUTO_PACKAGE_UTILS_RDEPENDS + + automatically added to each package run-time dependency variable RDEPENDS_foo-X. + +In the bzip2 case, all the utilities depend on the libbz2.so shared library, so we have:: + + AUTO_PACKAGE_UTILS_RDEPENDS = "libbz2" + +.. + TODO: Document AUTO_PACKAGE_UTILS_PACKAGES and + AUTO_PACKAGE_UTILS_PROVIDES. They seem to be used for the same + purpose, namely making the main package (or an extra special-purpose + foo-utils package) RDEPEND on all the utilities. In practice, + there's probably no difference between + + RDEPENDS_${PN} += "${AUTO_PACKAGE_UTILS_PROVIDES}" + + and + + RDEPENDS_${PN} += "${AUTO_PACKAGE_UTILS_PACKAGES}" From 44817c969ba557fe14048996595cf978d38a1a3d Mon Sep 17 00:00:00 2001 From: Rasmus Villemoes Date: Thu, 22 Dec 2016 14:22:32 +0100 Subject: [PATCH 09/10] recipes.rst: document pkgconfig class --- source/recipes.rst | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/source/recipes.rst b/source/recipes.rst index 2b7194c..c746a36 100644 --- a/source/recipes.rst +++ b/source/recipes.rst @@ -416,6 +416,17 @@ the recipe file to force a serial build. thus serves to ensure that the ``DESTDIR`` variable is defined appropriately. +.. _pkgconfig_class: + +pkgconfig +--------- + +Many autotools configure scripts rely on pkg-config to figure out the +proper compiler/linker flags for various needed libraries (or to +detect if those libraries are even present). To make sure pkg-config +picks up libraries and headers from the staging area rather than the +host, you should inherit the :oe:cls:`pkgconfig` class. + .. _auto_package_libs_class: auto-package-libs From cae17e744bd403906a805c8c7358a722423564e8 Mon Sep 17 00:00:00 2001 From: Rasmus Villemoes Date: Fri, 23 Dec 2016 15:14:38 +0100 Subject: [PATCH 10/10] recipes.rst: begin kernel.oeclass documentation This still needs a lot of work, but it's probably still better than nothing. There are some comments which don't appear in the compiled output, but which I think should be addressed at some point. --- source/recipes.rst | 75 +++++++++++++++++++++++++++++++++++++++++++++- source/tasks.rst | 8 ++--- 2 files changed, 78 insertions(+), 5 deletions(-) diff --git a/source/recipes.rst b/source/recipes.rst index c746a36..cb3a652 100644 --- a/source/recipes.rst +++ b/source/recipes.rst @@ -298,7 +298,7 @@ first word of the name be the same as the recipe name. A USE flag example ~~~~~~~~~~~~~~~~~~ -An example from the `freetype` recipe:: +An example from the freetype recipe:: RECIPE_FLAGS += "freetype_bzip2" EXTRA_OECONF += "${EXTRA_OECONF_BZIP2}" @@ -599,3 +599,76 @@ In the bzip2 case, all the utilities depend on the libbz2.so shared library, so and RDEPENDS_${PN} += "${AUTO_PACKAGE_UTILS_PACKAGES}" + +.. _kernel_class: + +kernel +------ + +Different boards have different requirements for how to build the +Linux kernel and associated paraphernalia (device tree blob, modules, +initramfs, ...). The kernel class provides the logic needed for a wide +range of scenarios, but one of course still needs to define the +various requirements. + +The first step in building a linux kernel is to configure it. The USE +flag ``USE_kernel_defconfig`` can either name an in-tree defconfig +target (e.g. ``bcm2835_defconfig``), or be set to the special value +``file``. In the latter case, one is supposed to provide a +configuration file in SRC_URI. The file ``${DEFCONFIG_FILE}`` is then +be copied to ``.config`` and ``make olddefconfig`` is run to set +config variables that were not defined in the provided file. The +default value of ``${DEFCONFIG_FILE}`` is ``${SRCDIR}/defconfig``. + +Next, one may choose to include an initramfs image in the kernel image +itself. To do that, set the USE flag ``USE_kernel_initramfs`` to +``True``. The kernel config variable ``CONFIG_INITRAMFS_SOURCE`` +(determining the source(s) from which the initramfs is built) is set +to the value of the variable ``${KERNEL_INITRAMFS}``, whose default +value is ``${MACHINE_SYSROOT}${bootdir}/initramfs.cpio``. + +The USE flag ``USE_ramdisk_image_compression`` governs how the initrd +image is compressed. Its default value is ``none``; other meaningful +values are ``gzip``, ``lzma``, ``xz``. + +.. + XXX: I'm not sure about that. We set CONFIG_INITRAMFS_COMPRESSION_*, + but those don't exist since 3.13 (more precisely 9ba4bcb64589). The + compression type seems to be determined from the CONFIG_RD_* + variables, and, from reading usr/Makefile, it seems that the last + defined among bz2, lzma, xz, lzo, lz4, gz wins, and they're all + "default y". + +Next, for compiling the kernel image itself, one chooses the generated +image type via the USE flag ``USE_kernel_imagetype``, whose default +value is ``zImage``. If one sets this to ``uImage``, one should also +define the load address by setting the USE flag +``USE_kernel_uimage_loadaddress``. The do_compile step compiles both +the kernel image as well as all kernel modules (if CONFIG_MODULES is +set, of course). + +.. + It seems that there's also a boolean USE_kernel_uimage which, if + set, causes do_compile_kernel_uimage to be run. That function seems + to open-code more or less the same thing as the kernel's build + system does when one says "make uImage LOADADDR=...". There's also + obviously some copy-pasted logic between that function and + do_compile_kernel to determine a default LOADADDR/ENTRYPOINT, though + they're not completely equivalent. Since only kernel_imagetype is + documented, and no machine config I can find defined + USE_kernel_uimage, I'm going to ignore that USE flag and the + associated postfunc. + + In do_compile_kernel, USE_kernel_uimage_loadaddress and + USE_kernel_uimage_entrypoint are equivalent, with the former having + precedence, so the latter is a redundant synonym, which is why I + don't mention it. + +Most embedded targets need a device tree blob. Unsurprisingly, +configuring which device tree source is used (if any) and the device +tree compiler to use is controlled by a number of USE flags. Setting +``USE_kernel_dts`` to the name of an in-tree ``.dts`` file causes that +file to be compiled; setting ``USE_kernel_dts`` to the special value +"1" is equivalent to setting it to ``${MACHINE}.dts``. Alternatively, +one can set ``USE_kernel_external_dtb`` to the name of an already +compiled device tree blob which one provides via ``SRC_URI``. diff --git a/source/tasks.rst b/source/tasks.rst index 04fe026..d3e8145 100644 --- a/source/tasks.rst +++ b/source/tasks.rst @@ -235,10 +235,10 @@ the ``FILES_${PN}-dev`` package. Each file under ``${D}`` (that is, the files generated by the `do_install_task` task) belongs to one and only one package. Packages are processed in the order they appear in the :oe:var:`PACKAGES` -variable, and the first package with a `FILES_*` glob pattern matching -a given file gets it. This means that when creating custom packages, -one usually uses the prepend operator ``=+`` to add the new package -name. +variable, and the first package with a ``FILES_*`` glob pattern +matching a given file gets it. This means that when creating custom +packages, one usually uses the prepend operator ``=+`` to add the new +package name. If, after processing all packages, there are files not claimed by any package, the do_split task fails.