feat: big endian c codegen

changes.rst

@@ -1,5 +1,23 @@
 .. currentmodule:: bitproto
 
+Version 1.3.0
+-------------
+
+.. _version-1.3.0:
+
+- Feature: Big-endian host support. bitproto's wire format is little-endian on
+  every platform; the C library (standard mode) and the C optimization-mode
+  generated code now also produce correct output when compiled for a big-endian
+  host. Python and Go were already endian-neutral and unaffected.
+- Feature: New compiler option ``--endian`` (``both`` | ``little`` | ``big``)
+  controlling the host byte order targeted by optimization-mode C/C++ code.
+  The default ``both`` emits both code paths guarded by ``#ifdef BP_BIG_ENDIAN``
+  and auto-detects the host (overridable by defining ``BP_BIG_ENDIAN``);
+  ``little`` keeps only the original, smaller and faster byte-pointer code (no
+  big-endian support); ``big`` emits only the portable path.
+- The little-endian generated code is unchanged from previous releases, so there
+  is no performance regression on the common little-endian target.
+
 Version 1.2.2
 -------------
 

compiler/bitproto/__init__.py

@@ -8,5 +8,5 @@
 
 """
 
-__version__ = "1.2.2"
+__version__ = "1.3.0"
 __description__ = "bit level data interchange format."

compiler/bitproto/_main.py

@@ -84,6 +84,21 @@ def build_arg_parser() -> argparse.ArgumentParser:
             "this works only if optimization mode enabled."
         ),
     )
+    args_parser.add_argument(
+        "--endian",
+        dest="endian",
+        type=str,
+        choices=["both", "little", "big"],
+        default="both",
+        help=(
+            "host byte order to target for optimization-mode C/C++ code. "
+            "'both' (default) emits both paths guarded by #ifdef BP_BIG_ENDIAN "
+            "and auto-detects the host; 'little' emits only the fast "
+            "little-endian path (smaller output, no big-endian support); "
+            "'big' emits only the portable big-endian path. "
+            "Only affects optimization mode."
+        ),
+    )
     args_parser.add_argument(
         "-v",
         "--version",
@@ -115,6 +130,7 @@ def run_bitproto() -> None:
         check=args.check,
         enable_optimize=args.enable_optimize,
         filter_messages=filter_messages,
+        endian=args.endian,
     )
 
 
@@ -126,6 +142,7 @@ def main(
     check: bool = False,
     enable_optimize: bool = False,
     filter_messages: Optional[List[str]] = None,
+    endian: str = "both",
 ) -> None:
     # Parse
     try:
@@ -161,6 +178,7 @@ def main(
             outdir=outdir,
             optimization_mode=enable_optimize,
             optimization_mode_filter_messages=filter_messages,
+            optimization_mode_endian=endian,
         )
     except RendererError as error:
         fatal(error.colored())

compiler/bitproto/renderer/__init__.py

@@ -18,6 +18,7 @@ def render(
     outdir: Optional[str] = None,
     optimization_mode: bool = False,
     optimization_mode_filter_messages: Optional[List[str]] = None,
+    optimization_mode_endian: str = "both",
 ) -> List[str]:
     """Render given `proto` to directory `outdir`.
     Returns the filepath list generated.
@@ -33,6 +34,7 @@ def render(
             outdir=outdir,
             optimization_mode=optimization_mode,
             optimization_mode_filter_messages=optimization_mode_filter_messages,
+            optimization_mode_endian=optimization_mode_endian,
         )
         outs.append(renderer.render())
     return outs

compiler/bitproto/renderer/block.py

@@ -59,6 +59,9 @@ class BlockRenderContext(Generic[F]):
     formatter: F
     bound: Proto
     optimization_mode_filter_messages: Optional[List[str]] = None
+    # One of "both" / "little" / "big": which host-endianness path(s) the
+    # optimization-mode C/C++ codegen should emit. See bitproto._main.
+    optimization_mode_endian: str = "both"
 
 
 class Block(Generic[F]):

compiler/bitproto/renderer/impls/c/formatter.py

@@ -315,34 +315,133 @@ def format_bp_array_json_formatter_name_from_alias(self, t: Array, d: Alias) ->
     def format_op_mode_endecoder_message_var(self) -> str:
         return "(*m)"
 
+    def _format_unsigned_chain_type(self, t: Type) -> str:
+        """Returns the unsigned C integer type for the field type t.
+        Used to generate endian-neutral bit-shift code instead of byte-pointer
+        indexing, which assumes little-endian host byte order.
+        """
+        if isinstance(t, Alias):
+            return self._format_unsigned_chain_type(t.type)
+        if isinstance(t, Enum):
+            return self._format_unsigned_chain_type(t.type)
+        if isinstance(t, Bool):
+            return "uint8_t"
+        if isinstance(t, Byte):
+            return "unsigned char"
+        if isinstance(t, Uint):
+            return self.format_uint_type(t)
+        if isinstance(t, Int):
+            n = self.get_nbits_of_integer(t)
+            return f"uint{n}_t"
+        return "unsigned"
+
+    # When True, the op-mode item formatters emit endian-neutral, value-based
+    # bit-shift code (correct on big-endian hosts).  When False (default), they
+    # emit the faster byte-pointer indexing code, which assumes little-endian
+    # host byte order.  The C renderer emits both variants guarded by
+    # #ifdef BP_BIG_ENDIAN so little-endian builds keep the fast path.
+    _op_mode_big_endian: bool = False
+
+    def format_op_mode_message_endian(
+        self, message: Message, is_encode: bool, big_endian: bool
+    ) -> List[str]:
+        """Generate the op-mode encode/decode statements for a message, choosing
+        the little-endian (fast) or big-endian (portable) item variant.
+        """
+        self._op_mode_big_endian = big_endian
+        try:
+            if is_encode:
+                return self.format_op_mode_encode_message(message)
+            return self.format_op_mode_decode_message(message)
+        finally:
+            self._op_mode_big_endian = False
+
     @override(Formatter)
     def format_op_mode_encoder_item(
         self, chain: str, t: Type, si: int, fi: int, shift: int, mask: int, r: int
     ) -> str:
-        """Implements format_op_mode_encoder_item for C.
+        if self._op_mode_big_endian:
+            return self._format_op_mode_encoder_item_be(
+                chain, t, si, fi, shift, mask, r
+            )
+        return self._format_op_mode_encoder_item_le(chain, t, si, fi, shift, mask, r)
+
+    def _format_op_mode_encoder_item_le(
+        self, chain: str, t: Type, si: int, fi: int, shift: int, mask: int, r: int
+    ) -> str:
+        """Little-endian (fast) encoder item, using byte-pointer indexing.
         Generated C statement like:
 
             s[0] = (((unsigned char *)&((*m).color))[0] >> 3) & 7;
-
         """
         assign = "=" if r == 0 else "|="
         shift_s = self.format_op_mode_smart_shift(shift)
         return f"s[{si}] {assign} (((unsigned char *)&({chain}))[{fi}] {shift_s}) & {mask};"
 
+    def _format_op_mode_encoder_item_be(
+        self, chain: str, t: Type, si: int, fi: int, shift: int, mask: int, r: int
+    ) -> str:
+        """Big-endian (portable) encoder item, using explicit bit shifts instead
+        of byte-pointer indexing so the output is correct on big-endian hosts.
+        fi*8 scales the byte index into a bit offset; the per-byte shift is then
+        folded in so a single right- or left-shift covers both.
+        Generated C statement like:
+
+            s[0] = ((uint8_t)((*m).color) >> 3) & 7;
+        """
+        assign = "=" if r == 0 else "|="
+        unsigned_type = self._format_unsigned_chain_type(t)
+        total_shift = fi * 8 + shift
+        total_shift_s = self.format_op_mode_smart_shift(total_shift)
+        return f"s[{si}] {assign} (({unsigned_type})({chain}){total_shift_s}) & {mask};"
+
     @override(Formatter)
     def format_op_mode_decoder_item(
         self, chain: str, t: Type, si: int, fi: int, shift: int, mask: int, r: int
     ) -> str:
-        """Implements format_op_mode_decoder_item for C.
+        if self._op_mode_big_endian:
+            return self._format_op_mode_decoder_item_be(
+                chain, t, si, fi, shift, mask, r
+            )
+        return self._format_op_mode_decoder_item_le(chain, t, si, fi, shift, mask, r)
+
+    def _format_op_mode_decoder_item_le(
+        self, chain: str, t: Type, si: int, fi: int, shift: int, mask: int, r: int
+    ) -> str:
+        """Little-endian (fast) decoder item, using byte-pointer indexing.
+        Uses = on the first write to a byte (r == 0) so no memset is required.
         Generated C statement like:
 
             ((unsigned char *)&((*m).color))[0] = (s[0] << 3) & 7;
-
         """
         assign = "=" if r == 0 else "|="
         shift_s = self.format_op_mode_smart_shift(shift)
         return f"((unsigned char *)&({chain}))[{fi}] {assign} (s[{si}] {shift_s}) & {mask};"
 
+    def _format_op_mode_decoder_item_be(
+        self, chain: str, t: Type, si: int, fi: int, shift: int, mask: int, r: int
+    ) -> str:
+        """Big-endian (portable) decoder item.
+        Generated C statement like:
+
+            (*m).color |= (uint8_t)(((unsigned)(s[0]) << 3) & 7);
+
+        Always uses |= (never =) so each byte contributes its bits without
+        clobbering already-written bytes; the big-endian Decode branch starts
+        with memset(m, 0, sizeof(*m)) to guarantee a zero baseline.
+        fi*8 positions the extracted bits at the correct byte of the field.
+        """
+        chain_type = self.format_type(t)
+        shift_s = self.format_op_mode_smart_shift(shift)
+        fi_shift = fi * 8
+        byte_val = f"(((unsigned)(s[{si}]){shift_s}) & {mask})"
+        if fi_shift > 0:
+            unsigned_type = self._format_unsigned_chain_type(t)
+            return (
+                f"{chain} |= ({chain_type})(({unsigned_type}){byte_val} << {fi_shift});"
+            )
+        return f"{chain} |= ({chain_type}){byte_val};"
+
     @override(Formatter)
     def post_format_op_mode_endecode_single_type(
         self, t: Type, chain: str, is_encode: bool

compiler/bitproto/renderer/impls/c/renderer_c.py

@@ -407,29 +407,93 @@ class BlockIncludeOpMode(Block[F]):
     def render(self) -> None:
         header_filename = self.formatter.format_out_filename(self.bound, extension=".h")
         self.push(f'#include "{header_filename}"')
+        endian = self._get_ctx_or_raise().optimization_mode_endian
+        if endian == "both":
+            # Auto-detect a big-endian host so the default just works without
+            # the user defining BP_BIG_ENDIAN; still overridable via -D.
+            self.push("#if !defined(BP_BIG_ENDIAN) && defined(__BYTE_ORDER__) && \\")
+            self.push("    (__BYTE_ORDER__ == __ORDER_BIG_ENDIAN__)")
+            self.push("#define BP_BIG_ENDIAN 1")
+            self.push("#endif")
+            # <string.h> is only needed for memset in the big-endian path.
+            self.push("#ifdef BP_BIG_ENDIAN")
+            self.push("#include <string.h>")
+            self.push("#endif")
+        elif endian == "big":
+            self.push("#include <string.h>")
 
 
 class BlockMessageEncoderOpMode(BlockMessageEncoderBase):
     @override(Block)
     def render(self) -> None:
+        endian = self._get_ctx_or_raise().optimization_mode_endian
         self.push(f"{self.function_signature} {{")
-        l = self.formatter.format_op_mode_encode_message(self.d)
-        for line in l:
-            self.push(line, indent=4)
+        self._push_body(endian)
         self.push("return 0;", indent=4)
         self.push("}")
 
+    def _push_body(self, endian: str) -> None:
+        def le() -> None:
+            for line in self.formatter.format_op_mode_message_endian(
+                self.d, is_encode=True, big_endian=False
+            ):
+                self.push(line, indent=4)
+
+        def be() -> None:
+            for line in self.formatter.format_op_mode_message_endian(
+                self.d, is_encode=True, big_endian=True
+            ):
+                self.push(line, indent=4)
+
+        if endian == "little":
+            le()
+        elif endian == "big":
+            be()
+        else:  # both: fast little-endian path, portable big-endian behind #else.
+            self.push("#ifndef BP_BIG_ENDIAN")
+            le()
+            self.push("#else")
+            be()
+            self.push("#endif")
+
 
 class BlockMessageDecoderOpMode(BlockMessageDecoderBase):
     @override(Block)
     def render(self) -> None:
+        endian = self._get_ctx_or_raise().optimization_mode_endian
         self.push(f"{self.function_signature} {{")
-        l = self.formatter.format_op_mode_decode_message(self.d)
-        for line in l:
-            self.push(line, indent=4)
+        self._push_body(endian)
         self.push("return 0;", indent=4)
         self.push("}")
 
+    def _push_body(self, endian: str) -> None:
+        # The little-endian path uses = on the first byte write, so it needs no
+        # zeroing.  The big-endian path always uses |=, so it must start from a
+        # zeroed message (memset).
+        def le() -> None:
+            for line in self.formatter.format_op_mode_message_endian(
+                self.d, is_encode=False, big_endian=False
+            ):
+                self.push(line, indent=4)
+
+        def be() -> None:
+            self.push("memset(m, 0, sizeof(*m));", indent=4)
+            for line in self.formatter.format_op_mode_message_endian(
+                self.d, is_encode=False, big_endian=True
+            ):
+                self.push(line, indent=4)
+
+        if endian == "little":
+            le()
+        elif endian == "big":
+            be()
+        else:  # both
+            self.push("#ifndef BP_BIG_ENDIAN")
+            le()
+            self.push("#else")
+            be()
+            self.push("#endif")
+
 
 class BlockMessageFunctionsOpMode(BlockBindMessage[F], BlockComposition[F]):
     @override(BlockComposition)

compiler/bitproto/renderer/renderer.py

@@ -31,12 +31,14 @@ def __init__(
         outdir: Optional[str] = None,
         optimization_mode: bool = False,
         optimization_mode_filter_messages: Optional[List[str]] = None,
+        optimization_mode_endian: str = "both",
     ) -> None:
         self.proto = proto
         self.outdir = outdir or self.get_outdir_default(proto)
         self.optimization_mode = optimization_mode
         self.check_proto_for_optimization_mode()
         self.optimization_mode_filter_messages = optimization_mode_filter_messages
+        self.optimization_mode_endian = optimization_mode_endian
 
         self.out_filename = self.get_out_filename()
         self.out_filepath = os.path.join(self.outdir, self.out_filename)
@@ -66,6 +68,7 @@ def render_string(self) -> str:
             formatter=formatter,
             bound=self.proto,
             optimization_mode_filter_messages=self.optimization_mode_filter_messages,
+            optimization_mode_endian=self.optimization_mode_endian,
         )
         block._render_with_ctx(ctx)
         return block._collect()

docs/compiler.rst

@@ -73,3 +73,11 @@ bitproto file doesn't meet the :ref:`style-guide`, to disable the linter:
 .. sourcecode:: bash
 
    $ bitproto c proto.bitproto -q
+
+In :ref:`optimization mode <performance-optimization-mode>` (``-O``), the ``--endian`` option
+controls the host byte order targeted by the generated C/C++ code, one of ``both`` (default),
+``little`` or ``big``. See :ref:`endianness`.
+
+.. sourcecode:: bash
+
+   $ bitproto c proto.bitproto -O --endian=little