diff --git a/extern_standalone/extern_standalone.c b/extern_standalone/extern_standalone.c new file mode 100644 index 0000000000000..467a6ce70c5c9 --- /dev/null +++ b/extern_standalone/extern_standalone.c @@ -0,0 +1,1074 @@ +/**************************************************************************/ +/* */ +/* OCaml */ +/* */ +/* Xavier Leroy, projet Cristal, INRIA Rocquencourt */ +/* */ +/* Copyright 1996 Institut National de Recherche en Informatique et */ +/* en Automatique. */ +/* */ +/* All rights reserved. This file is distributed under the terms of */ +/* the GNU Lesser General Public License version 2.1, with the */ +/* special exception on linking described in the file LICENSE. */ +/* */ +/**************************************************************************/ + +/* Standalone version of the marshalling ("extern") code from + runtime/extern.c (OxCaml runtime), i.e. the writer side of the Marshal + standard library module. All required definitions from the OCaml + runtime headers (mlvalues.h, gc.h, intext.h, misc.h, config.h/m.h) are + inlined below; only standard C headers are included, although a + compiler providing GCC-style builtins (GCC, Clang) is required. + + Values passed to the entry points must use the native-code OxCaml value + representation: 64-bit words, blocks preceded by a one-word header with + 8 reserved bits, 46 size bits, 2 colour bits and 8 tag bits. Header + colour bits are ignored. The reachable-words machinery that also lives + in runtime/extern.c is not included here. + + Differences from runtime/extern.c: + - 64-bit platforms only (an OxCaml requirement anyway). Support for + reading marshalled data back on 32-bit platforms has also been + removed: the COMPAT_32 flag is gone and the header field giving the + size in words when read on a 32-bit platform is written as 0 (64-bit + readers do not consult it). + - Compression (the COMPRESSED flag and the associated zstd hook) is + not supported. + - The caml_serialize_* functions for user-defined marshallers are + omitted. Consequently custom blocks (Custom_tag: Int32, Int64, + Nativeint, Bigarray, ...) cannot be marshalled, since their + serialization functions would have no way to write output; they are + rejected like other abstract values. + - Byte-swapping is done with the __builtin_bswap* builtins rather + than the runtime's hand-written shifts and byte copies. + - Errors are reported by returning a [caml_extern_error] code (with the + message available via [caml_extern_error_message]) instead of raising + OCaml exceptions. Internally this uses setjmp/longjmp, mirroring + the runtime's raise-through behaviour. + - Marshalling flags are not supported: sharing is always preserved, + as with an empty flag list passed to the Marshal functions. + - The only entry point is caml_output_value_to_malloc; the other + entry points of the runtime version (channel-based, to OCaml bytes, + or to a caller-supplied buffer) are omitted. + - Closures cannot be marshalled, since the runtime's code-fragment + table is not available here. Like continuations and custom, + abstract and mixed blocks, they cause + CAML_EXTERN_ERROR_INVALID_ARGUMENT to be returned. + - Memory is allocated with malloc/calloc/free instead of caml_stat_*. + - The extern state is a single global instead of being per-domain + (Caml_state->extern_state); this code is therefore not thread-safe. + - Function names are kept from the runtime, so this file cannot be + linked into a program that also links the OCaml runtime. */ + +#include +#include +#include +#include +#include +#include + +/* --------------------------------------------------------------------- */ +/* Platform configuration (from caml/config.h and the generated caml/m.h) */ + +#if UINTPTR_MAX != 0xFFFFFFFFFFFFFFFFull +#error "extern_standalone.c only supports 64-bit platforms" +#endif + +#if defined(__BYTE_ORDER__) && __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ +#define ARCH_BIG_ENDIAN +#endif + +/* Doubles are assumed to be IEEE 754 and to have the same byte order as + integers. This holds on all supported 64-bit platforms; the runtime's + ARCH_FLOAT_ENDIANNESS mixed-endian configurations only existed on + 32-bit ARM. */ + +/* From caml/misc.h */ + +typedef intptr_t intnat; +typedef uintptr_t uintnat; +typedef size_t asize_t; + +static inline int caml_umul_overflow(uintnat a, uintnat b, uintnat * res) +{ + return __builtin_mul_overflow(a, b, res); +} + +/* --------------------------------------------------------------------- */ +/* Value representation (from caml/mlvalues.h) */ + +typedef intnat value; +typedef uintnat header_t; +typedef uintnat mlsize_t; +typedef unsigned int tag_t; + +#define Is_null(v) ((v) == 0) + +static inline int Is_long(value x) { + return ((x & 1) != 0 || x == 0); +} + +static inline int Is_block(value x) { + return ((x & 1) == 0 && x != 0); +} + +#define Long_val(x) ((x) >> 1) + +/* Headers: 8 reserved bits (nonzero only for mixed blocks), then 46 + size bits, 2 colour bits and 8 tag bits (see the top of this file). + In the runtime Hd_val is a relaxed atomic load. Make_header uses + the colour for out-of-heap blocks (NOT_MARKABLE, i.e. 3) and zero + reserved bits. */ + +#define Hd_val(val) (((volatile header_t *) (val)) [-1]) +#define Tag_hd(hd) ((tag_t) ((hd) & 0xFF)) +#define Wosize_hd(hd) \ + ((mlsize_t) (((hd) >> 10) & (((header_t) 1 << 46) - 1))) +#define Reserved_hd(hd) ((hd) >> 56) +#define Tag_val(val) Tag_hd(Hd_val(val)) +#define Wosize_val(val) Wosize_hd(Hd_val(val)) + +#define Make_header(wosize, tag) \ + (((header_t) (wosize) << 10) + (3 << 8) + (tag)) + +#define Field(x, i) (((volatile value *)(x)) [i]) /* Also an l-value. */ +#define Forward_val(v) Field(v, 0) +#define String_val(x) ((const char *) (x)) + +#define Forcing_tag 244 +#define Cont_tag 245 +#define Lazy_tag 246 +#define Closure_tag 247 +#define Infix_tag 249 +#define Forward_tag 250 +#define Abstract_tag 251 +#define String_tag 252 +#define Double_tag 253 +#define Double_array_tag 254 +#define Custom_tag 255 + +/* From caml/str.c */ + +static mlsize_t caml_string_length(value s) +{ + const char * p = (const char *) s; + mlsize_t temp = Wosize_val(s) * sizeof(value) - 1; + assert (p[temp - p[temp]] == 0); + return temp - p[temp]; +} + +/* --------------------------------------------------------------------- */ +/* The marshal format (from caml/intext.h) */ + +/* Magic number */ + +#define Intext_magic_number_small 0x8495A6BE +#define Intext_magic_number_big 0x8495A6BF + +/* Header format for the "small" model: 20 bytes + 0 "small" magic number + 4 length of marshaled data, in bytes + 8 number of shared blocks + 12 size in words when read on a 32-bit platform + (always 0 here: 32-bit readers are not supported) + 16 size in words when read on a 64-bit platform + The 4 numbers are 32 bits each, in big endian. + + Header format for the "big" model: 32 bytes + 0 "big" magic number + 4 four reserved bytes, currently set to 0 + 8 length of marshaled data, in bytes + 16 number of shared blocks + 24 size in words when read on a 64-bit platform + The 3 numbers are 64 bits each, in big endian. +*/ + +#define MAX_INTEXT_HEADER_SIZE 32 + +/* Codes for the compact format */ + +#define PREFIX_SMALL_BLOCK 0x80 +#define PREFIX_SMALL_INT 0x40 +#define PREFIX_SMALL_STRING 0x20 +#define CODE_INT8 0x0 +#define CODE_INT16 0x1 +#define CODE_INT32 0x2 +#define CODE_INT64 0x3 +#define CODE_SHARED8 0x4 +#define CODE_SHARED16 0x5 +#define CODE_SHARED32 0x6 +#define CODE_SHARED64 0x14 +#define CODE_BLOCK32 0x8 +#define CODE_BLOCK64 0x13 +#define CODE_STRING8 0x9 +#define CODE_STRING32 0xA +#define CODE_STRING64 0x15 +#define CODE_DOUBLE_BIG 0xB +#define CODE_DOUBLE_LITTLE 0xC +#define CODE_DOUBLE_ARRAY8_BIG 0xD +#define CODE_DOUBLE_ARRAY8_LITTLE 0xE +#define CODE_DOUBLE_ARRAY32_BIG 0xF +#define CODE_DOUBLE_ARRAY32_LITTLE 0x7 +#define CODE_DOUBLE_ARRAY64_BIG 0x16 +#define CODE_DOUBLE_ARRAY64_LITTLE 0x17 +#define CODE_CODEPOINTER 0x10 +#define CODE_INFIXPOINTER 0x11 +#define OLD_CODE_CUSTOM 0x12 // no longer supported +#define CODE_CUSTOM_LEN 0x18 +#define CODE_CUSTOM_FIXED 0x19 + +#define CODE_UNBOXED_INT64 0x1a // Jane Street extensions +#define CODE_NULL 0x1f + +#ifdef ARCH_BIG_ENDIAN +#define CODE_DOUBLE_NATIVE CODE_DOUBLE_BIG +#define CODE_DOUBLE_ARRAY8_NATIVE CODE_DOUBLE_ARRAY8_BIG +#define CODE_DOUBLE_ARRAY32_NATIVE CODE_DOUBLE_ARRAY32_BIG +#define CODE_DOUBLE_ARRAY64_NATIVE CODE_DOUBLE_ARRAY64_BIG +#else +#define CODE_DOUBLE_NATIVE CODE_DOUBLE_LITTLE +#define CODE_DOUBLE_ARRAY8_NATIVE CODE_DOUBLE_ARRAY8_LITTLE +#define CODE_DOUBLE_ARRAY32_NATIVE CODE_DOUBLE_ARRAY32_LITTLE +#define CODE_DOUBLE_ARRAY64_NATIVE CODE_DOUBLE_ARRAY64_LITTLE +#endif + +/* Size-ing data structures for extern. Chosen so that + sizeof(struct output_block) is slightly below 8Kb. */ + +#define SIZE_EXTERN_OUTPUT_BLOCK 8100 + +struct caml_output_block { + struct caml_output_block * next; + char * end; + char data[SIZE_EXTERN_OUTPUT_BLOCK]; +}; + +/* --------------------------------------------------------------------- */ +/* Public interface of this file */ + +/* Errors. The runtime raises Out_of_memory / Invalid_argument; here + the error is returned and the corresponding message (a string + literal) is available via [caml_extern_error_message]. */ + +typedef enum { + CAML_EXTERN_OK = 0, + CAML_EXTERN_ERROR_OUT_OF_MEMORY = 1, + CAML_EXTERN_ERROR_INVALID_ARGUMENT = 2 +} caml_extern_error; + +const char * caml_extern_error_message(void); + +caml_extern_error caml_output_value_to_malloc + (value v, /*out*/ char ** buf, /*out*/ intnat * len); + /* Marshal [v] to a buffer allocated with malloc. On success returns + CAML_EXTERN_OK and stores the buffer and its length in bytes in + [*buf] and [*len]; the caller is responsible for freeing the + buffer. On failure returns an error code and writes neither + [*buf] nor [*len]. Sharing within [v] is always preserved, as + with an empty flag list in the runtime. Values with no external + representation (closures, continuations, custom / abstract / mixed + blocks) yield CAML_EXTERN_ERROR_INVALID_ARGUMENT. */ + +void caml_free_extern_state (void); + /* Free the internal state (lazily allocated on first use and then + reused across calls). */ + +/* --------------------------------------------------------------------- */ +/* The marshaller itself (from runtime/extern.c) */ + +/* Stack for pending values to marshal */ + +#define EXTERN_STACK_INIT_SIZE 256 +#define EXTERN_STACK_MAX_SIZE (1024*1024*100) + +struct extern_item { volatile value * v; mlsize_t count; }; + +/* Hash table to record already-marshaled objects and their positions */ + +struct object_position { value obj; uintnat pos; }; + +/* The hash table uses open addressing, linear probing, and a redundant + representation: + - a bitvector [present] records which entries of the table are occupied; + - an array [entries] records (object, position) pairs for the entries + that are occupied. + The bitvector is much smaller than the array (1/128th on 64-bit + platforms, 1/64th on 32-bit platforms), so it has better locality, + making it faster to determine that an object is not in the table. + Also, it makes it faster to empty or initialize a table: only the + [present] bitvector needs to be filled with zeros, the [entries] + array can be left uninitialized. +*/ + +struct position_table { + int shift; + mlsize_t size; /* size == 1 << (wordsize - shift) */ + mlsize_t mask; /* mask == size - 1 */ + mlsize_t threshold; /* threshold == a fixed fraction of size */ + uintnat * present; /* [Bitvect_size(size)] */ + struct object_position * entries; /* [size] */ +}; + +#define Bits_word (8 * sizeof(uintnat)) +#define Bitvect_size(n) (((n) + Bits_word - 1) / Bits_word) + +#define POS_TABLE_INIT_SIZE_LOG2 8 +#define POS_TABLE_INIT_SIZE (1 << POS_TABLE_INIT_SIZE_LOG2) + +struct caml_extern_state { + + uintnat obj_counter; /* Number of objects emitted so far */ + uintnat size_64; /* Size in words of 64-bit block for struct. */ + + /* Return point for error reporting (replaces raising OCaml + exceptions in the runtime version) */ + jmp_buf error_return; + + /* Stack for pending value to marshal */ + struct extern_item extern_stack_init[EXTERN_STACK_INIT_SIZE]; + struct extern_item * extern_stack; + struct extern_item * extern_stack_limit; + + /* Hash table to record already marshalled objects */ + uintnat pos_table_present_init[Bitvect_size(POS_TABLE_INIT_SIZE)]; + struct object_position pos_table_entries_init[POS_TABLE_INIT_SIZE]; + struct position_table pos_table; + + /* To buffer the output */ + + char * extern_ptr; + char * extern_limit; + + struct caml_output_block * extern_output_first; + struct caml_output_block * extern_output_block; +}; + +/* The extern state. The runtime keeps this per-domain in + Caml_state->extern_state; here it is a single global. */ + +static struct caml_extern_state * extern_state = NULL; + +/* Error reporting, replacing the exception raising of the runtime + version. All messages are string literals. */ + +static caml_extern_error extern_error_code = CAML_EXTERN_OK; +static const char * extern_error_msg = NULL; + +const char * caml_extern_error_message(void) +{ + return extern_error_msg; +} + +_Noreturn static void extern_raise(struct caml_extern_state* s, + caml_extern_error code, const char * msg) +{ + extern_error_code = code; + extern_error_msg = msg; + longjmp(s->error_return, 1); +} + +static void init_extern_stack(struct caml_extern_state* s) +{ + /* (Re)initialize the globals for next time around */ + s->extern_stack = s->extern_stack_init; + s->extern_stack_limit = s->extern_stack + EXTERN_STACK_INIT_SIZE; +} + +/* Returns NULL on out-of-memory (the runtime version raises). */ +static struct caml_extern_state* init_extern_state (void) +{ + struct caml_extern_state* s; + + if (extern_state != NULL) + return extern_state; + + s = malloc(sizeof(struct caml_extern_state)); + if (s == NULL) return NULL; + + s->obj_counter = 0; + s->size_64 = 0; + init_extern_stack(s); + + extern_state = s; + return s; +} + +void caml_free_extern_state (void) +{ + if (extern_state != NULL) { + free(extern_state); + extern_state = NULL; + } +} + +/* Forward declarations */ + +_Noreturn static void extern_out_of_memory(struct caml_extern_state* s); + +_Noreturn static +void extern_invalid_argument(struct caml_extern_state* s, + const char *msg); + +_Noreturn static void extern_stack_overflow(struct caml_extern_state* s); + +static void free_extern_output(struct caml_extern_state* s); + +static void extern_free_stack(struct caml_extern_state* s) +{ + /* Free the extern stack if needed */ + if (s->extern_stack != s->extern_stack_init) { + free(s->extern_stack); + init_extern_stack(s); + } +} + +static struct extern_item * extern_resize_stack(struct caml_extern_state* s, + const struct extern_item * sp) +{ + asize_t newsize = 2 * (s->extern_stack_limit - s->extern_stack); + asize_t sp_offset = sp - s->extern_stack; + struct extern_item * newstack; + + if (newsize >= EXTERN_STACK_MAX_SIZE) extern_stack_overflow(s); + newstack = calloc(newsize, sizeof(struct extern_item)); + if (newstack == NULL) extern_stack_overflow(s); + + /* Copy items from the old stack to the new stack */ + memcpy (newstack, s->extern_stack, + sizeof(struct extern_item) * sp_offset); + + /* Free the old stack if it is not the initial stack */ + if (s->extern_stack != s->extern_stack_init) + free(s->extern_stack); + + s->extern_stack = newstack; + s->extern_stack_limit = newstack + newsize; + return newstack + sp_offset; +} + +/* Multiplicative Fibonacci hashing + (Knuth, TAOCP vol 3, section 6.4, page 518). + HASH_FACTOR is (sqrt(5) - 1) / 2 * 2^wordsize. */ +#define HASH_FACTOR 11400714819323198486UL +#define Hash(v,shift) (((uintnat)(v) * HASH_FACTOR) >> (shift)) + +/* When the table becomes 2/3 full, its size is increased. */ +#define Threshold(sz) (((sz) * 2) / 3) + +/* Initialize the position table */ + +static void extern_init_position_table(struct caml_extern_state* s) +{ + s->pos_table.size = POS_TABLE_INIT_SIZE; + s->pos_table.shift = 8 * sizeof(value) - POS_TABLE_INIT_SIZE_LOG2; + s->pos_table.mask = POS_TABLE_INIT_SIZE - 1; + s->pos_table.threshold = Threshold(POS_TABLE_INIT_SIZE); + s->pos_table.present = s->pos_table_present_init; + s->pos_table.entries = s->pos_table_entries_init; + memset(s->pos_table_present_init, 0, + Bitvect_size(POS_TABLE_INIT_SIZE) * sizeof(uintnat)); +} + +/* Free the position table */ + +static void extern_free_position_table(struct caml_extern_state* s) +{ + if (s->pos_table.present != s->pos_table_present_init) { + free(s->pos_table.present); + free(s->pos_table.entries); + /* Protect against repeated calls to extern_free_position_table */ + s->pos_table.present = s->pos_table_present_init; + s->pos_table.entries = s->pos_table_entries_init; + } +} + +/* Accessing bitvectors */ + +static inline uintnat bitvect_test(uintnat * bv, uintnat i) +{ + return bv[i / Bits_word] & ((uintnat) 1 << (i & (Bits_word - 1))); +} + +static inline void bitvect_set(uintnat * bv, uintnat i) +{ + bv[i / Bits_word] |= ((uintnat) 1 << (i & (Bits_word - 1))); +} + +/* Grow the position table */ + +static void extern_resize_position_table(struct caml_extern_state *s) +{ + mlsize_t new_size, new_byte_size; + int new_shift; + uintnat * new_present; + struct object_position * new_entries; + uintnat h; + struct position_table old = s->pos_table; + + /* Grow the table quickly (x 8) up to 10^6 entries, + more slowly (x 2) afterwards. */ + if (old.size < 1000000) { + new_size = 8 * old.size; + new_shift = old.shift - 3; + } else { + new_size = 2 * old.size; + new_shift = old.shift - 1; + } + if (new_size == 0 + || caml_umul_overflow(new_size, sizeof(struct object_position), + &new_byte_size)) + extern_out_of_memory(s); + new_entries = malloc(new_byte_size); + if (new_entries == NULL) extern_out_of_memory(s); + new_present = + calloc(Bitvect_size(new_size), sizeof(uintnat)); + if (new_present == NULL) { + free(new_entries); + extern_out_of_memory(s); + } + s->pos_table.size = new_size; + s->pos_table.shift = new_shift; + s->pos_table.mask = new_size - 1; + s->pos_table.threshold = Threshold(new_size); + s->pos_table.present = new_present; + s->pos_table.entries = new_entries; + + /* Insert every entry of the old table in the new table */ + for (uintnat i = 0; i < old.size; i++) { + if (! bitvect_test(old.present, i)) continue; + h = Hash(old.entries[i].obj, s->pos_table.shift); + while (bitvect_test(new_present, h)) { + h = (h + 1) & s->pos_table.mask; + } + bitvect_set(new_present, h); + new_entries[h] = old.entries[i]; + } + + /* Free the old tables if they are not the initial ones */ + if (old.present != s->pos_table_present_init) { + free(old.present); + free(old.entries); + } +} + +/* Determine whether the given object [obj] is in the hash table. + If so, set [*pos_out] to its position in the output and return 1. + If not, return 0. + Either way, set [*h_out] to the hash value appropriate for + [extern_record_location]. */ +static inline int extern_lookup_position(struct caml_extern_state *s, value obj, + uintnat * pos_out, uintnat * h_out) +{ + uintnat h = Hash(obj, s->pos_table.shift); + while (1) { + if (! bitvect_test(s->pos_table.present, h)) { + *h_out = h; + return 0; + } + if (s->pos_table.entries[h].obj == obj) { + *h_out = h; + *pos_out = s->pos_table.entries[h].pos; + return 1; + } + h = (h + 1) & s->pos_table.mask; + } +} + +/* Record the output position for the given object [obj]. */ +/* The [h] parameter is the index in the hash table where the object + must be inserted. It was determined during lookup. */ +static void extern_record_location(struct caml_extern_state* s, + value obj, uintnat h) +{ + bitvect_set(s->pos_table.present, h); + s->pos_table.entries[h].obj = obj; + s->pos_table.entries[h].pos = s->obj_counter; + s->obj_counter++; + if (s->obj_counter >= s->pos_table.threshold) + extern_resize_position_table(s); +} + +/* To buffer the output */ + +static void init_extern_output(struct caml_extern_state* s) +{ + s->extern_output_first = + malloc(sizeof(struct caml_output_block)); + if (s->extern_output_first == NULL) + extern_raise(s, CAML_EXTERN_ERROR_OUT_OF_MEMORY, + "output_value: out of memory"); + s->extern_output_block = s->extern_output_first; + s->extern_output_block->next = NULL; + s->extern_ptr = s->extern_output_block->data; + s->extern_limit = s->extern_output_block->data + SIZE_EXTERN_OUTPUT_BLOCK; +} + +static void close_extern_output(struct caml_extern_state* s) +{ + s->extern_output_block->end = s->extern_ptr; +} + +static void free_extern_output(struct caml_extern_state* s) +{ + for (struct caml_output_block *blk = s->extern_output_first, *nextblk; + blk != NULL; + blk = nextblk) { + nextblk = blk->next; + free(blk); + } + s->extern_output_first = NULL; + extern_free_stack(s); + extern_free_position_table(s); +} + +static void grow_extern_output(struct caml_extern_state *s, intnat required) +{ + struct caml_output_block * blk; + intnat extra; + + s->extern_output_block->end = s->extern_ptr; + if (required <= SIZE_EXTERN_OUTPUT_BLOCK / 2) + extra = 0; + else + extra = required; + blk = malloc(sizeof(struct caml_output_block) + extra); + if (blk == NULL) extern_out_of_memory(s); + s->extern_output_block->next = blk; + s->extern_output_block = blk; + s->extern_output_block->next = NULL; + s->extern_ptr = s->extern_output_block->data; + s->extern_limit = + s->extern_output_block->data + SIZE_EXTERN_OUTPUT_BLOCK + extra; +} + +static intnat extern_output_length(struct caml_extern_state* s) +{ + struct caml_output_block * blk; + intnat len; + + for (len = 0, blk = s->extern_output_first; blk != NULL; blk = blk->next) + len += blk->end - blk->data; + return len; +} + +/* Error raising, with cleanup */ + +static void extern_out_of_memory(struct caml_extern_state* s) +{ + free_extern_output(s); + extern_raise(s, CAML_EXTERN_ERROR_OUT_OF_MEMORY, + "output_value: out of memory"); +} + +static void extern_invalid_argument(struct caml_extern_state *s, + const char *msg) +{ + free_extern_output(s); + extern_raise(s, CAML_EXTERN_ERROR_INVALID_ARGUMENT, msg); +} + +static void extern_stack_overflow(struct caml_extern_state* s) +{ + free_extern_output(s); + extern_raise(s, CAML_EXTERN_ERROR_OUT_OF_MEMORY, + "output_value: stack overflow"); +} + +/* Conversion to big-endian */ + +#ifdef ARCH_BIG_ENDIAN +#define Htobe16(x) ((uint16_t) (x)) +#define Htobe32(x) ((uint32_t) (x)) +#define Htobe64(x) ((uint64_t) (x)) +#else +#define Htobe16(x) __builtin_bswap16((uint16_t) (x)) +#define Htobe32(x) __builtin_bswap32((uint32_t) (x)) +#define Htobe64(x) __builtin_bswap64((uint64_t) (x)) +#endif + +static inline void store16(char * dst, int n) +{ + uint16_t u = Htobe16(n); + memcpy(dst, &u, 2); +} + +static inline void store32(char * dst, intnat n) +{ + uint32_t u = Htobe32(n); + memcpy(dst, &u, 4); +} + +static inline void store64(char * dst, int64_t n) +{ + uint64_t u = Htobe64(n); + memcpy(dst, &u, 8); +} + +/* Write characters, integers, and blocks in the output buffer */ + +static inline void writebyte(struct caml_extern_state* s, int c) +{ + if (s->extern_ptr >= s->extern_limit) grow_extern_output(s, 1); + *s->extern_ptr++ = c; +} + +static void writeblock(struct caml_extern_state* s, const char * data, + intnat len) +{ + if (s->extern_ptr + len > s->extern_limit) grow_extern_output(s, len); + memcpy(s->extern_ptr, data, len); + s->extern_ptr += len; +} + +static inline void writeblock_float8(struct caml_extern_state* s, + const double * data, intnat ndoubles) +{ + /* Doubles have the same byte order as integers, so they can be + written natively; the CODE_DOUBLE*_NATIVE codes record which + endianness that is. (The runtime additionally handles mixed-endian + doubles here, which only exist on 32-bit ARM.) */ + writeblock(s, (const char *) data, ndoubles * 8); +} + +static void writecode8(struct caml_extern_state* s, + int code, intnat val) +{ + if (s->extern_ptr + 2 > s->extern_limit) grow_extern_output(s, 2); + s->extern_ptr[0] = code; + s->extern_ptr[1] = val; + s->extern_ptr += 2; +} + +static void writecode16(struct caml_extern_state* s, + int code, intnat val) +{ + if (s->extern_ptr + 3 > s->extern_limit) grow_extern_output(s, 3); + s->extern_ptr[0] = code; + store16(s->extern_ptr + 1, (int) val); + s->extern_ptr += 3; +} + +static void writecode32(struct caml_extern_state* s, + int code, intnat val) +{ + if (s->extern_ptr + 5 > s->extern_limit) grow_extern_output(s, 5); + s->extern_ptr[0] = code; + store32(s->extern_ptr + 1, val); + s->extern_ptr += 5; +} + +static void writecode64(struct caml_extern_state* s, + int code, intnat val) +{ + if (s->extern_ptr + 9 > s->extern_limit) grow_extern_output(s, 9); + s->extern_ptr[0] = code; + store64(s->extern_ptr + 1, val); + s->extern_ptr += 9; +} + +/* Marshaling integers */ + +static inline void extern_int(struct caml_extern_state* s, intnat n) +{ + if (n >= 0 && n < 0x40) { + writebyte(s, PREFIX_SMALL_INT + n); + } else if (n >= -(1 << 7) && n < (1 << 7)) { + writecode8(s, CODE_INT8, n); + } else if (n >= -(1 << 15) && n < (1 << 15)) { + writecode16(s, CODE_INT16, n); + } else if (n < -((intnat)1 << 30) || n >= ((intnat)1 << 30)) { + writecode64(s, CODE_INT64, n); + } else { + writecode32(s, CODE_INT32, n); + } +} + +static inline void extern_null(struct caml_extern_state* s) +{ + writecode8(s, CODE_NULL, 0); +} + +/* Marshaling references to previously-marshaled blocks */ + +static inline void extern_shared_reference(struct caml_extern_state* s, + uintnat d) +{ + if (d < 0x100) { + writecode8(s, CODE_SHARED8, d); + } else if (d < 0x10000) { + writecode16(s, CODE_SHARED16, d); + } else if (d >= (uintnat)1 << 32) { + writecode64(s, CODE_SHARED64, d); + } else { + writecode32(s, CODE_SHARED32, d); + } +} + +/* Marshaling block headers */ + +static inline void extern_header(struct caml_extern_state* s, + mlsize_t sz, tag_t tag) +{ + if (tag < 16 && sz < 8) { + writebyte(s, PREFIX_SMALL_BLOCK + tag + (sz << 4)); + } else { + header_t hd = Make_header(sz, tag); + if (hd < (uintnat)1 << 32) + writecode32(s, CODE_BLOCK32, hd); + else + writecode64(s, CODE_BLOCK64, hd); + } +} + +/* Marshaling strings */ + +static inline void extern_string(struct caml_extern_state *s, + value v, mlsize_t len) +{ + if (len < 0x20) { + writebyte(s, PREFIX_SMALL_STRING + len); + } else if (len < 0x100) { + writecode8(s, CODE_STRING8, len); + } else { + if (len < (uintnat)1 << 32) + writecode32(s, CODE_STRING32, len); + else + writecode64(s, CODE_STRING64, len); + } + writeblock(s, String_val(v), len); +} + +/* Marshaling FP numbers */ + +static inline void extern_double(struct caml_extern_state* s, value v) +{ + writebyte(s, CODE_DOUBLE_NATIVE); + writeblock_float8(s, (double *) v, 1); +} + +/* Marshaling FP arrays */ + +static inline void extern_double_array(struct caml_extern_state* s, + value v, mlsize_t nfloats) +{ + if (nfloats < 0x100) { + writecode8(s, CODE_DOUBLE_ARRAY8_NATIVE, nfloats); + } else { + if (nfloats < (uintnat) 1 << 32) + writecode32(s, CODE_DOUBLE_ARRAY32_NATIVE, nfloats); + else + writecode64(s, CODE_DOUBLE_ARRAY64_NATIVE, nfloats); + } + writeblock_float8(s, (double *) v, nfloats); +} + +/* Marshal the given value in the output buffer */ + +static void extern_rec(struct caml_extern_state* s, value v) +{ + struct extern_item * sp; + uintnat h = 0; + uintnat pos = 0; + + /* for Double_tag and Double_array_tag */ + static_assert(sizeof(double) == 8, ""); + + extern_init_position_table(s); + sp = s->extern_stack; + + while(1) { + if (Is_null(v)) { + extern_null(s); + } else if (Is_long(v)) { + extern_int(s, Long_val(v)); + } + else { + header_t hd = Hd_val(v); + tag_t tag = Tag_hd(hd); + mlsize_t sz = Wosize_hd(hd); + if (Reserved_hd(hd) != 0) { + /* Nonzero reserved header bits indicate a mixed block. */ + extern_invalid_argument(s, "output_value: mixed block"); + break; + } + + if (tag == Forward_tag) { + value f = Forward_val (v); + if (Is_block (f) + && ( Tag_val (f) == Forward_tag + || Tag_val (f) == Lazy_tag + || Tag_val (f) == Forcing_tag + /* Double_tag check because of flat float arrays */ + || Tag_val (f) == Double_tag + )){ + /* Do not short-circuit the pointer. */ + }else{ + v = f; + continue; + } + } + /* Atoms are treated specially for two reasons: they are not allocated + in the externed block, and they are automatically shared. */ + if (sz == 0) { + extern_header(s, 0, tag); + goto next_item; + } + /* Check if object already seen */ + if (extern_lookup_position(s, v, &pos, &h)) { + extern_shared_reference(s, s->obj_counter - pos); + goto next_item; + } + /* Output the contents of the object */ + switch(tag) { + case String_tag: { + mlsize_t len = caml_string_length(v); + extern_string(s, v, len); + s->size_64 += 1 + (len + 8) / 8; + extern_record_location(s, v, h); + break; + } + case Double_tag: { + extern_double(s, v); + s->size_64 += 1 + 1; + extern_record_location(s, v, h); + break; + } + case Double_array_tag: { + /* sizeof(double) == sizeof(value), per the static_assert above */ + mlsize_t nfloats = Wosize_val(v); + extern_double_array(s, v, nfloats); + s->size_64 += 1 + nfloats; + extern_record_location(s, v, h); + break; + } + case Abstract_tag: + extern_invalid_argument(s, "output_value: abstract value (Abstract)"); + break; + case Infix_tag: + /* An infix pointer into a closure block; closures cannot be + marshalled (see Closure_tag below). */ + extern_invalid_argument(s, "output_value: functional value"); + break; + case Custom_tag: + /* Custom blocks (Int32, Int64, Nativeint, Bigarray, ...) cannot + be marshalled here: their serialization functions would need the + caml_serialize_* entry points, which are not provided. */ + extern_invalid_argument(s, "output_value: abstract value (Custom)"); + break; + case Closure_tag: + /* The runtime's code-fragment table is not available here, so + code pointers, and hence closures, cannot be marshalled. */ + extern_invalid_argument(s, "output_value: functional value"); + break; + case Cont_tag: + extern_invalid_argument(s, "output_value: continuation value"); + break; + default: { + extern_header(s, sz, tag); + s->size_64 += 1 + sz; + extern_record_location(s, v, h); + /* Remember that we still have to serialize fields 1 ... sz - 1 */ + if (sz > 1) { + sp++; + if (sp >= s->extern_stack_limit) + sp = extern_resize_stack(s, sp); + sp->v = &Field(v, 1); + sp->count = sz - 1; + } + /* Continue serialization with the first field */ + v = Field(v, 0); + continue; + } + } + } + next_item: + /* Pop one more item to marshal, if any */ + if (sp == s->extern_stack) { + /* We are done. Cleanup the stack and leave the function */ + extern_free_stack(s); + extern_free_position_table(s); + return; + } + v = *((sp->v)++); + if (--(sp->count) == 0) sp--; + } + /* Never reached as function leaves with return */ +} + +static intnat extern_value(struct caml_extern_state* s, value v, + /*out*/ char header[MAX_INTEXT_HEADER_SIZE], + /*out*/ int * header_len) +{ + intnat res_len; + /* Initializations */ + s->obj_counter = 0; + s->size_64 = 0; + /* Marshal the object */ + extern_rec(s, v); + /* Record end of output */ + close_extern_output(s); + /* Write the header */ + res_len = extern_output_length(s); + if (res_len >= ((intnat)1 << 32) || s->size_64 >= ((intnat)1 << 32)) { + /* The object is too big for the small header format. + Use the big header. */ + store32(header, Intext_magic_number_big); + store32(header + 4, 0); + store64(header + 8, res_len); + store64(header + 16, s->obj_counter); + store64(header + 24, s->size_64); + *header_len = 32; + return res_len; + } + /* Use the small header format. The field at offset 12 is the size in + words when read on a 32-bit platform; 32-bit readers are not + supported, so 0 is written instead. */ + store32(header, Intext_magic_number_small); + store32(header + 4, res_len); + store32(header + 8, s->obj_counter); + store32(header + 12, 0); + store32(header + 16, s->size_64); + *header_len = 20; + return res_len; +} + +/* The entry point */ + +caml_extern_error +caml_output_value_to_malloc(value v, + /*out*/ char ** buf, + /*out*/ intnat * len) +{ + char header[MAX_INTEXT_HEADER_SIZE]; + int header_len; + intnat data_len; + char * res; + struct caml_extern_state* s = init_extern_state (); + + if (s == NULL) { + extern_error_code = CAML_EXTERN_ERROR_OUT_OF_MEMORY; + extern_error_msg = "output_value: out of memory"; + return extern_error_code; + } + extern_error_msg = NULL; + if (setjmp(s->error_return) != 0) return extern_error_code; + + init_extern_output(s); + data_len = extern_value(s, v, header, &header_len); + res = malloc(header_len + data_len); + if (res == NULL) extern_out_of_memory(s); + *buf = res; + *len = header_len + data_len; + memcpy(res, header, header_len); + res += header_len; + for (struct caml_output_block *blk = s->extern_output_first, *nextblk; + blk != NULL; + blk = nextblk) { + intnat n = blk->end - blk->data; + memcpy(res, blk->data, n); + res += n; + nextblk = blk->next; + free(blk); + } + return CAML_EXTERN_OK; +} diff --git a/extern_standalone/extern_standalone.cpp b/extern_standalone/extern_standalone.cpp new file mode 100644 index 0000000000000..cdef3d61364dc --- /dev/null +++ b/extern_standalone/extern_standalone.cpp @@ -0,0 +1,1086 @@ +/**************************************************************************/ +/* */ +/* OCaml */ +/* */ +/* Xavier Leroy, projet Cristal, INRIA Rocquencourt */ +/* */ +/* Copyright 1996 Institut National de Recherche en Informatique et */ +/* en Automatique. */ +/* */ +/* All rights reserved. This file is distributed under the terms of */ +/* the GNU Lesser General Public License version 2.1, with the */ +/* special exception on linking described in the file LICENSE. */ +/* */ +/**************************************************************************/ + +/* Standalone version of the marshalling ("extern") code from + runtime/extern.c (OxCaml runtime), i.e. the writer side of the Marshal + standard library module. All required definitions from the OCaml + runtime headers (mlvalues.h, gc.h, intext.h, misc.h, config.h/m.h) are + inlined below; only standard C headers are included, although a + compiler providing GCC-style builtins (GCC, Clang) is required. + + This file is the C++ counterpart of extern_standalone.c. It differs + only in using [[noreturn]] instead of [[noreturn]], casting the results + of malloc/calloc, and giving the public interface extern "C" linkage + (so that this build can be linked with C code such as + extern_standalone_test.c). + + Values passed to the entry points must use the native-code OxCaml value + representation: 64-bit words, blocks preceded by a one-word header with + 8 reserved bits, 46 size bits, 2 colour bits and 8 tag bits. Header + colour bits are ignored. The reachable-words machinery that also lives + in runtime/extern.c is not included here. + + Differences from runtime/extern.c: + - 64-bit platforms only (an OxCaml requirement anyway). Support for + reading marshalled data back on 32-bit platforms has also been + removed: the COMPAT_32 flag is gone and the header field giving the + size in words when read on a 32-bit platform is written as 0 (64-bit + readers do not consult it). + - Compression (the COMPRESSED flag and the associated zstd hook) is + not supported. + - The caml_serialize_* functions for user-defined marshallers are + omitted. Consequently custom blocks (Custom_tag: Int32, Int64, + Nativeint, Bigarray, ...) cannot be marshalled, since their + serialization functions would have no way to write output; they are + rejected like other abstract values. + - Byte-swapping is done with the __builtin_bswap* builtins rather + than the runtime's hand-written shifts and byte copies. + - Errors are reported by returning a [caml_extern_error] code (with the + message available via [caml_extern_error_message]) instead of raising + OCaml exceptions. Internally this uses setjmp/longjmp, mirroring + the runtime's raise-through behaviour. + - Marshalling flags are not supported: sharing is always preserved, + as with an empty flag list passed to the Marshal functions. + - The only entry point is caml_output_value_to_malloc; the other + entry points of the runtime version (channel-based, to OCaml bytes, + or to a caller-supplied buffer) are omitted. + - Closures cannot be marshalled, since the runtime's code-fragment + table is not available here. Like continuations and custom, + abstract and mixed blocks, they cause + CAML_EXTERN_ERROR_INVALID_ARGUMENT to be returned. + - Memory is allocated with malloc/calloc/free instead of caml_stat_*. + - The extern state is a single global instead of being per-domain + (Caml_state->extern_state); this code is therefore not thread-safe. + - Function names are kept from the runtime, so this file cannot be + linked into a program that also links the OCaml runtime. */ + +#include +#include +#include +#include +#include +#include + +/* --------------------------------------------------------------------- */ +/* Platform configuration (from caml/config.h and the generated caml/m.h) */ + +#if UINTPTR_MAX != 0xFFFFFFFFFFFFFFFFull +#error "extern_standalone.cpp only supports 64-bit platforms" +#endif + +#if defined(__BYTE_ORDER__) && __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ +#define ARCH_BIG_ENDIAN +#endif + +/* Doubles are assumed to be IEEE 754 and to have the same byte order as + integers. This holds on all supported 64-bit platforms; the runtime's + ARCH_FLOAT_ENDIANNESS mixed-endian configurations only existed on + 32-bit ARM. */ + +/* From caml/misc.h */ + +typedef intptr_t intnat; +typedef uintptr_t uintnat; +typedef size_t asize_t; + +static inline int caml_umul_overflow(uintnat a, uintnat b, uintnat * res) +{ + return __builtin_mul_overflow(a, b, res); +} + +/* --------------------------------------------------------------------- */ +/* Value representation (from caml/mlvalues.h) */ + +typedef intnat value; +typedef uintnat header_t; +typedef uintnat mlsize_t; +typedef unsigned int tag_t; + +#define Is_null(v) ((v) == 0) + +static inline int Is_long(value x) { + return ((x & 1) != 0 || x == 0); +} + +static inline int Is_block(value x) { + return ((x & 1) == 0 && x != 0); +} + +#define Long_val(x) ((x) >> 1) + +/* Headers: 8 reserved bits (nonzero only for mixed blocks), then 46 + size bits, 2 colour bits and 8 tag bits (see the top of this file). + In the runtime Hd_val is a relaxed atomic load. Make_header uses + the colour for out-of-heap blocks (NOT_MARKABLE, i.e. 3) and zero + reserved bits. */ + +#define Hd_val(val) (((volatile header_t *) (val)) [-1]) +#define Tag_hd(hd) ((tag_t) ((hd) & 0xFF)) +#define Wosize_hd(hd) \ + ((mlsize_t) (((hd) >> 10) & (((header_t) 1 << 46) - 1))) +#define Reserved_hd(hd) ((hd) >> 56) +#define Tag_val(val) Tag_hd(Hd_val(val)) +#define Wosize_val(val) Wosize_hd(Hd_val(val)) + +#define Make_header(wosize, tag) \ + (((header_t) (wosize) << 10) + (3 << 8) + (tag)) + +#define Field(x, i) (((volatile value *)(x)) [i]) /* Also an l-value. */ +#define Forward_val(v) Field(v, 0) +#define String_val(x) ((const char *) (x)) + +#define Forcing_tag 244 +#define Cont_tag 245 +#define Lazy_tag 246 +#define Closure_tag 247 +#define Infix_tag 249 +#define Forward_tag 250 +#define Abstract_tag 251 +#define String_tag 252 +#define Double_tag 253 +#define Double_array_tag 254 +#define Custom_tag 255 + +/* From caml/str.c */ + +static mlsize_t caml_string_length(value s) +{ + const char * p = (const char *) s; + mlsize_t temp = Wosize_val(s) * sizeof(value) - 1; + assert (p[temp - p[temp]] == 0); + return temp - p[temp]; +} + +/* --------------------------------------------------------------------- */ +/* The marshal format (from caml/intext.h) */ + +/* Magic number */ + +#define Intext_magic_number_small 0x8495A6BE +#define Intext_magic_number_big 0x8495A6BF + +/* Header format for the "small" model: 20 bytes + 0 "small" magic number + 4 length of marshaled data, in bytes + 8 number of shared blocks + 12 size in words when read on a 32-bit platform + (always 0 here: 32-bit readers are not supported) + 16 size in words when read on a 64-bit platform + The 4 numbers are 32 bits each, in big endian. + + Header format for the "big" model: 32 bytes + 0 "big" magic number + 4 four reserved bytes, currently set to 0 + 8 length of marshaled data, in bytes + 16 number of shared blocks + 24 size in words when read on a 64-bit platform + The 3 numbers are 64 bits each, in big endian. +*/ + +#define MAX_INTEXT_HEADER_SIZE 32 + +/* Codes for the compact format */ + +#define PREFIX_SMALL_BLOCK 0x80 +#define PREFIX_SMALL_INT 0x40 +#define PREFIX_SMALL_STRING 0x20 +#define CODE_INT8 0x0 +#define CODE_INT16 0x1 +#define CODE_INT32 0x2 +#define CODE_INT64 0x3 +#define CODE_SHARED8 0x4 +#define CODE_SHARED16 0x5 +#define CODE_SHARED32 0x6 +#define CODE_SHARED64 0x14 +#define CODE_BLOCK32 0x8 +#define CODE_BLOCK64 0x13 +#define CODE_STRING8 0x9 +#define CODE_STRING32 0xA +#define CODE_STRING64 0x15 +#define CODE_DOUBLE_BIG 0xB +#define CODE_DOUBLE_LITTLE 0xC +#define CODE_DOUBLE_ARRAY8_BIG 0xD +#define CODE_DOUBLE_ARRAY8_LITTLE 0xE +#define CODE_DOUBLE_ARRAY32_BIG 0xF +#define CODE_DOUBLE_ARRAY32_LITTLE 0x7 +#define CODE_DOUBLE_ARRAY64_BIG 0x16 +#define CODE_DOUBLE_ARRAY64_LITTLE 0x17 +#define CODE_CODEPOINTER 0x10 +#define CODE_INFIXPOINTER 0x11 +#define OLD_CODE_CUSTOM 0x12 // no longer supported +#define CODE_CUSTOM_LEN 0x18 +#define CODE_CUSTOM_FIXED 0x19 + +#define CODE_UNBOXED_INT64 0x1a // Jane Street extensions +#define CODE_NULL 0x1f + +#ifdef ARCH_BIG_ENDIAN +#define CODE_DOUBLE_NATIVE CODE_DOUBLE_BIG +#define CODE_DOUBLE_ARRAY8_NATIVE CODE_DOUBLE_ARRAY8_BIG +#define CODE_DOUBLE_ARRAY32_NATIVE CODE_DOUBLE_ARRAY32_BIG +#define CODE_DOUBLE_ARRAY64_NATIVE CODE_DOUBLE_ARRAY64_BIG +#else +#define CODE_DOUBLE_NATIVE CODE_DOUBLE_LITTLE +#define CODE_DOUBLE_ARRAY8_NATIVE CODE_DOUBLE_ARRAY8_LITTLE +#define CODE_DOUBLE_ARRAY32_NATIVE CODE_DOUBLE_ARRAY32_LITTLE +#define CODE_DOUBLE_ARRAY64_NATIVE CODE_DOUBLE_ARRAY64_LITTLE +#endif + +/* Size-ing data structures for extern. Chosen so that + sizeof(struct output_block) is slightly below 8Kb. */ + +#define SIZE_EXTERN_OUTPUT_BLOCK 8100 + +struct caml_output_block { + struct caml_output_block * next; + char * end; + char data[SIZE_EXTERN_OUTPUT_BLOCK]; +}; + +/* --------------------------------------------------------------------- */ +/* Public interface of this file */ + +/* Errors. The runtime raises Out_of_memory / Invalid_argument; here + the error is returned and the corresponding message (a string + literal) is available via [caml_extern_error_message]. */ + +typedef enum { + CAML_EXTERN_OK = 0, + CAML_EXTERN_ERROR_OUT_OF_MEMORY = 1, + CAML_EXTERN_ERROR_INVALID_ARGUMENT = 2 +} caml_extern_error; + +extern "C" { + +const char * caml_extern_error_message(void); + +caml_extern_error caml_output_value_to_malloc + (value v, /*out*/ char ** buf, /*out*/ intnat * len); + /* Marshal [v] to a buffer allocated with malloc. On success returns + CAML_EXTERN_OK and stores the buffer and its length in bytes in + [*buf] and [*len]; the caller is responsible for freeing the + buffer. On failure returns an error code and writes neither + [*buf] nor [*len]. Sharing within [v] is always preserved, as + with an empty flag list in the runtime. Values with no external + representation (closures, continuations, custom / abstract / mixed + blocks) yield CAML_EXTERN_ERROR_INVALID_ARGUMENT. */ + +void caml_free_extern_state (void); + /* Free the internal state (lazily allocated on first use and then + reused across calls). */ + +} /* extern "C" */ + +/* --------------------------------------------------------------------- */ +/* The marshaller itself (from runtime/extern.c) */ + +/* Stack for pending values to marshal */ + +#define EXTERN_STACK_INIT_SIZE 256 +#define EXTERN_STACK_MAX_SIZE (1024*1024*100) + +struct extern_item { volatile value * v; mlsize_t count; }; + +/* Hash table to record already-marshaled objects and their positions */ + +struct object_position { value obj; uintnat pos; }; + +/* The hash table uses open addressing, linear probing, and a redundant + representation: + - a bitvector [present] records which entries of the table are occupied; + - an array [entries] records (object, position) pairs for the entries + that are occupied. + The bitvector is much smaller than the array (1/128th on 64-bit + platforms, 1/64th on 32-bit platforms), so it has better locality, + making it faster to determine that an object is not in the table. + Also, it makes it faster to empty or initialize a table: only the + [present] bitvector needs to be filled with zeros, the [entries] + array can be left uninitialized. +*/ + +struct position_table { + int shift; + mlsize_t size; /* size == 1 << (wordsize - shift) */ + mlsize_t mask; /* mask == size - 1 */ + mlsize_t threshold; /* threshold == a fixed fraction of size */ + uintnat * present; /* [Bitvect_size(size)] */ + struct object_position * entries; /* [size] */ +}; + +#define Bits_word (8 * sizeof(uintnat)) +#define Bitvect_size(n) (((n) + Bits_word - 1) / Bits_word) + +#define POS_TABLE_INIT_SIZE_LOG2 8 +#define POS_TABLE_INIT_SIZE (1 << POS_TABLE_INIT_SIZE_LOG2) + +struct caml_extern_state { + + uintnat obj_counter; /* Number of objects emitted so far */ + uintnat size_64; /* Size in words of 64-bit block for struct. */ + + /* Return point for error reporting (replaces raising OCaml + exceptions in the runtime version) */ + jmp_buf error_return; + + /* Stack for pending value to marshal */ + struct extern_item extern_stack_init[EXTERN_STACK_INIT_SIZE]; + struct extern_item * extern_stack; + struct extern_item * extern_stack_limit; + + /* Hash table to record already marshalled objects */ + uintnat pos_table_present_init[Bitvect_size(POS_TABLE_INIT_SIZE)]; + struct object_position pos_table_entries_init[POS_TABLE_INIT_SIZE]; + struct position_table pos_table; + + /* To buffer the output */ + + char * extern_ptr; + char * extern_limit; + + struct caml_output_block * extern_output_first; + struct caml_output_block * extern_output_block; +}; + +/* The extern state. The runtime keeps this per-domain in + Caml_state->extern_state; here it is a single global. */ + +static struct caml_extern_state * extern_state = NULL; + +/* Error reporting, replacing the exception raising of the runtime + version. All messages are string literals. */ + +static caml_extern_error extern_error_code = CAML_EXTERN_OK; +static const char * extern_error_msg = NULL; + +const char * caml_extern_error_message(void) +{ + return extern_error_msg; +} + +[[noreturn]] static void extern_raise(struct caml_extern_state* s, + caml_extern_error code, const char * msg) +{ + extern_error_code = code; + extern_error_msg = msg; + longjmp(s->error_return, 1); +} + +static void init_extern_stack(struct caml_extern_state* s) +{ + /* (Re)initialize the globals for next time around */ + s->extern_stack = s->extern_stack_init; + s->extern_stack_limit = s->extern_stack + EXTERN_STACK_INIT_SIZE; +} + +/* Returns NULL on out-of-memory (the runtime version raises). */ +static struct caml_extern_state* init_extern_state (void) +{ + struct caml_extern_state* s; + + if (extern_state != NULL) + return extern_state; + + s = (struct caml_extern_state *) malloc(sizeof(struct caml_extern_state)); + if (s == NULL) return NULL; + + s->obj_counter = 0; + s->size_64 = 0; + init_extern_stack(s); + + extern_state = s; + return s; +} + +void caml_free_extern_state (void) +{ + if (extern_state != NULL) { + free(extern_state); + extern_state = NULL; + } +} + +/* Forward declarations */ + +[[noreturn]] static void extern_out_of_memory(struct caml_extern_state* s); + +[[noreturn]] static +void extern_invalid_argument(struct caml_extern_state* s, + const char *msg); + +[[noreturn]] static void extern_stack_overflow(struct caml_extern_state* s); + +static void free_extern_output(struct caml_extern_state* s); + +static void extern_free_stack(struct caml_extern_state* s) +{ + /* Free the extern stack if needed */ + if (s->extern_stack != s->extern_stack_init) { + free(s->extern_stack); + init_extern_stack(s); + } +} + +static struct extern_item * extern_resize_stack(struct caml_extern_state* s, + const struct extern_item * sp) +{ + asize_t newsize = 2 * (s->extern_stack_limit - s->extern_stack); + asize_t sp_offset = sp - s->extern_stack; + struct extern_item * newstack; + + if (newsize >= EXTERN_STACK_MAX_SIZE) extern_stack_overflow(s); + newstack = + (struct extern_item *) calloc(newsize, sizeof(struct extern_item)); + if (newstack == NULL) extern_stack_overflow(s); + + /* Copy items from the old stack to the new stack */ + memcpy (newstack, s->extern_stack, + sizeof(struct extern_item) * sp_offset); + + /* Free the old stack if it is not the initial stack */ + if (s->extern_stack != s->extern_stack_init) + free(s->extern_stack); + + s->extern_stack = newstack; + s->extern_stack_limit = newstack + newsize; + return newstack + sp_offset; +} + +/* Multiplicative Fibonacci hashing + (Knuth, TAOCP vol 3, section 6.4, page 518). + HASH_FACTOR is (sqrt(5) - 1) / 2 * 2^wordsize. */ +#define HASH_FACTOR 11400714819323198486UL +#define Hash(v,shift) (((uintnat)(v) * HASH_FACTOR) >> (shift)) + +/* When the table becomes 2/3 full, its size is increased. */ +#define Threshold(sz) (((sz) * 2) / 3) + +/* Initialize the position table */ + +static void extern_init_position_table(struct caml_extern_state* s) +{ + s->pos_table.size = POS_TABLE_INIT_SIZE; + s->pos_table.shift = 8 * sizeof(value) - POS_TABLE_INIT_SIZE_LOG2; + s->pos_table.mask = POS_TABLE_INIT_SIZE - 1; + s->pos_table.threshold = Threshold(POS_TABLE_INIT_SIZE); + s->pos_table.present = s->pos_table_present_init; + s->pos_table.entries = s->pos_table_entries_init; + memset(s->pos_table_present_init, 0, + Bitvect_size(POS_TABLE_INIT_SIZE) * sizeof(uintnat)); +} + +/* Free the position table */ + +static void extern_free_position_table(struct caml_extern_state* s) +{ + if (s->pos_table.present != s->pos_table_present_init) { + free(s->pos_table.present); + free(s->pos_table.entries); + /* Protect against repeated calls to extern_free_position_table */ + s->pos_table.present = s->pos_table_present_init; + s->pos_table.entries = s->pos_table_entries_init; + } +} + +/* Accessing bitvectors */ + +static inline uintnat bitvect_test(uintnat * bv, uintnat i) +{ + return bv[i / Bits_word] & ((uintnat) 1 << (i & (Bits_word - 1))); +} + +static inline void bitvect_set(uintnat * bv, uintnat i) +{ + bv[i / Bits_word] |= ((uintnat) 1 << (i & (Bits_word - 1))); +} + +/* Grow the position table */ + +static void extern_resize_position_table(struct caml_extern_state *s) +{ + mlsize_t new_size, new_byte_size; + int new_shift; + uintnat * new_present; + struct object_position * new_entries; + uintnat h; + struct position_table old = s->pos_table; + + /* Grow the table quickly (x 8) up to 10^6 entries, + more slowly (x 2) afterwards. */ + if (old.size < 1000000) { + new_size = 8 * old.size; + new_shift = old.shift - 3; + } else { + new_size = 2 * old.size; + new_shift = old.shift - 1; + } + if (new_size == 0 + || caml_umul_overflow(new_size, sizeof(struct object_position), + &new_byte_size)) + extern_out_of_memory(s); + new_entries = (struct object_position *) malloc(new_byte_size); + if (new_entries == NULL) extern_out_of_memory(s); + new_present = + (uintnat *) calloc(Bitvect_size(new_size), sizeof(uintnat)); + if (new_present == NULL) { + free(new_entries); + extern_out_of_memory(s); + } + s->pos_table.size = new_size; + s->pos_table.shift = new_shift; + s->pos_table.mask = new_size - 1; + s->pos_table.threshold = Threshold(new_size); + s->pos_table.present = new_present; + s->pos_table.entries = new_entries; + + /* Insert every entry of the old table in the new table */ + for (uintnat i = 0; i < old.size; i++) { + if (! bitvect_test(old.present, i)) continue; + h = Hash(old.entries[i].obj, s->pos_table.shift); + while (bitvect_test(new_present, h)) { + h = (h + 1) & s->pos_table.mask; + } + bitvect_set(new_present, h); + new_entries[h] = old.entries[i]; + } + + /* Free the old tables if they are not the initial ones */ + if (old.present != s->pos_table_present_init) { + free(old.present); + free(old.entries); + } +} + +/* Determine whether the given object [obj] is in the hash table. + If so, set [*pos_out] to its position in the output and return 1. + If not, return 0. + Either way, set [*h_out] to the hash value appropriate for + [extern_record_location]. */ +static inline int extern_lookup_position(struct caml_extern_state *s, value obj, + uintnat * pos_out, uintnat * h_out) +{ + uintnat h = Hash(obj, s->pos_table.shift); + while (1) { + if (! bitvect_test(s->pos_table.present, h)) { + *h_out = h; + return 0; + } + if (s->pos_table.entries[h].obj == obj) { + *h_out = h; + *pos_out = s->pos_table.entries[h].pos; + return 1; + } + h = (h + 1) & s->pos_table.mask; + } +} + +/* Record the output position for the given object [obj]. */ +/* The [h] parameter is the index in the hash table where the object + must be inserted. It was determined during lookup. */ +static void extern_record_location(struct caml_extern_state* s, + value obj, uintnat h) +{ + bitvect_set(s->pos_table.present, h); + s->pos_table.entries[h].obj = obj; + s->pos_table.entries[h].pos = s->obj_counter; + s->obj_counter++; + if (s->obj_counter >= s->pos_table.threshold) + extern_resize_position_table(s); +} + +/* To buffer the output */ + +static void init_extern_output(struct caml_extern_state* s) +{ + s->extern_output_first = + (struct caml_output_block *) malloc(sizeof(struct caml_output_block)); + if (s->extern_output_first == NULL) + extern_raise(s, CAML_EXTERN_ERROR_OUT_OF_MEMORY, + "output_value: out of memory"); + s->extern_output_block = s->extern_output_first; + s->extern_output_block->next = NULL; + s->extern_ptr = s->extern_output_block->data; + s->extern_limit = s->extern_output_block->data + SIZE_EXTERN_OUTPUT_BLOCK; +} + +static void close_extern_output(struct caml_extern_state* s) +{ + s->extern_output_block->end = s->extern_ptr; +} + +static void free_extern_output(struct caml_extern_state* s) +{ + for (struct caml_output_block *blk = s->extern_output_first, *nextblk; + blk != NULL; + blk = nextblk) { + nextblk = blk->next; + free(blk); + } + s->extern_output_first = NULL; + extern_free_stack(s); + extern_free_position_table(s); +} + +static void grow_extern_output(struct caml_extern_state *s, intnat required) +{ + struct caml_output_block * blk; + intnat extra; + + s->extern_output_block->end = s->extern_ptr; + if (required <= SIZE_EXTERN_OUTPUT_BLOCK / 2) + extra = 0; + else + extra = required; + blk = (struct caml_output_block *) + malloc(sizeof(struct caml_output_block) + extra); + if (blk == NULL) extern_out_of_memory(s); + s->extern_output_block->next = blk; + s->extern_output_block = blk; + s->extern_output_block->next = NULL; + s->extern_ptr = s->extern_output_block->data; + s->extern_limit = + s->extern_output_block->data + SIZE_EXTERN_OUTPUT_BLOCK + extra; +} + +static intnat extern_output_length(struct caml_extern_state* s) +{ + struct caml_output_block * blk; + intnat len; + + for (len = 0, blk = s->extern_output_first; blk != NULL; blk = blk->next) + len += blk->end - blk->data; + return len; +} + +/* Error raising, with cleanup */ + +static void extern_out_of_memory(struct caml_extern_state* s) +{ + free_extern_output(s); + extern_raise(s, CAML_EXTERN_ERROR_OUT_OF_MEMORY, + "output_value: out of memory"); +} + +static void extern_invalid_argument(struct caml_extern_state *s, + const char *msg) +{ + free_extern_output(s); + extern_raise(s, CAML_EXTERN_ERROR_INVALID_ARGUMENT, msg); +} + +static void extern_stack_overflow(struct caml_extern_state* s) +{ + free_extern_output(s); + extern_raise(s, CAML_EXTERN_ERROR_OUT_OF_MEMORY, + "output_value: stack overflow"); +} + +/* Conversion to big-endian */ + +#ifdef ARCH_BIG_ENDIAN +#define Htobe16(x) ((uint16_t) (x)) +#define Htobe32(x) ((uint32_t) (x)) +#define Htobe64(x) ((uint64_t) (x)) +#else +#define Htobe16(x) __builtin_bswap16((uint16_t) (x)) +#define Htobe32(x) __builtin_bswap32((uint32_t) (x)) +#define Htobe64(x) __builtin_bswap64((uint64_t) (x)) +#endif + +static inline void store16(char * dst, int n) +{ + uint16_t u = Htobe16(n); + memcpy(dst, &u, 2); +} + +static inline void store32(char * dst, intnat n) +{ + uint32_t u = Htobe32(n); + memcpy(dst, &u, 4); +} + +static inline void store64(char * dst, int64_t n) +{ + uint64_t u = Htobe64(n); + memcpy(dst, &u, 8); +} + +/* Write characters, integers, and blocks in the output buffer */ + +static inline void writebyte(struct caml_extern_state* s, int c) +{ + if (s->extern_ptr >= s->extern_limit) grow_extern_output(s, 1); + *s->extern_ptr++ = c; +} + +static void writeblock(struct caml_extern_state* s, const char * data, + intnat len) +{ + if (s->extern_ptr + len > s->extern_limit) grow_extern_output(s, len); + memcpy(s->extern_ptr, data, len); + s->extern_ptr += len; +} + +static inline void writeblock_float8(struct caml_extern_state* s, + const double * data, intnat ndoubles) +{ + /* Doubles have the same byte order as integers, so they can be + written natively; the CODE_DOUBLE*_NATIVE codes record which + endianness that is. (The runtime additionally handles mixed-endian + doubles here, which only exist on 32-bit ARM.) */ + writeblock(s, (const char *) data, ndoubles * 8); +} + +static void writecode8(struct caml_extern_state* s, + int code, intnat val) +{ + if (s->extern_ptr + 2 > s->extern_limit) grow_extern_output(s, 2); + s->extern_ptr[0] = code; + s->extern_ptr[1] = val; + s->extern_ptr += 2; +} + +static void writecode16(struct caml_extern_state* s, + int code, intnat val) +{ + if (s->extern_ptr + 3 > s->extern_limit) grow_extern_output(s, 3); + s->extern_ptr[0] = code; + store16(s->extern_ptr + 1, (int) val); + s->extern_ptr += 3; +} + +static void writecode32(struct caml_extern_state* s, + int code, intnat val) +{ + if (s->extern_ptr + 5 > s->extern_limit) grow_extern_output(s, 5); + s->extern_ptr[0] = code; + store32(s->extern_ptr + 1, val); + s->extern_ptr += 5; +} + +static void writecode64(struct caml_extern_state* s, + int code, intnat val) +{ + if (s->extern_ptr + 9 > s->extern_limit) grow_extern_output(s, 9); + s->extern_ptr[0] = code; + store64(s->extern_ptr + 1, val); + s->extern_ptr += 9; +} + +/* Marshaling integers */ + +static inline void extern_int(struct caml_extern_state* s, intnat n) +{ + if (n >= 0 && n < 0x40) { + writebyte(s, PREFIX_SMALL_INT + n); + } else if (n >= -(1 << 7) && n < (1 << 7)) { + writecode8(s, CODE_INT8, n); + } else if (n >= -(1 << 15) && n < (1 << 15)) { + writecode16(s, CODE_INT16, n); + } else if (n < -((intnat)1 << 30) || n >= ((intnat)1 << 30)) { + writecode64(s, CODE_INT64, n); + } else { + writecode32(s, CODE_INT32, n); + } +} + +static inline void extern_null(struct caml_extern_state* s) +{ + writecode8(s, CODE_NULL, 0); +} + +/* Marshaling references to previously-marshaled blocks */ + +static inline void extern_shared_reference(struct caml_extern_state* s, + uintnat d) +{ + if (d < 0x100) { + writecode8(s, CODE_SHARED8, d); + } else if (d < 0x10000) { + writecode16(s, CODE_SHARED16, d); + } else if (d >= (uintnat)1 << 32) { + writecode64(s, CODE_SHARED64, d); + } else { + writecode32(s, CODE_SHARED32, d); + } +} + +/* Marshaling block headers */ + +static inline void extern_header(struct caml_extern_state* s, + mlsize_t sz, tag_t tag) +{ + if (tag < 16 && sz < 8) { + writebyte(s, PREFIX_SMALL_BLOCK + tag + (sz << 4)); + } else { + header_t hd = Make_header(sz, tag); + if (hd < (uintnat)1 << 32) + writecode32(s, CODE_BLOCK32, hd); + else + writecode64(s, CODE_BLOCK64, hd); + } +} + +/* Marshaling strings */ + +static inline void extern_string(struct caml_extern_state *s, + value v, mlsize_t len) +{ + if (len < 0x20) { + writebyte(s, PREFIX_SMALL_STRING + len); + } else if (len < 0x100) { + writecode8(s, CODE_STRING8, len); + } else { + if (len < (uintnat)1 << 32) + writecode32(s, CODE_STRING32, len); + else + writecode64(s, CODE_STRING64, len); + } + writeblock(s, String_val(v), len); +} + +/* Marshaling FP numbers */ + +static inline void extern_double(struct caml_extern_state* s, value v) +{ + writebyte(s, CODE_DOUBLE_NATIVE); + writeblock_float8(s, (double *) v, 1); +} + +/* Marshaling FP arrays */ + +static inline void extern_double_array(struct caml_extern_state* s, + value v, mlsize_t nfloats) +{ + if (nfloats < 0x100) { + writecode8(s, CODE_DOUBLE_ARRAY8_NATIVE, nfloats); + } else { + if (nfloats < (uintnat) 1 << 32) + writecode32(s, CODE_DOUBLE_ARRAY32_NATIVE, nfloats); + else + writecode64(s, CODE_DOUBLE_ARRAY64_NATIVE, nfloats); + } + writeblock_float8(s, (double *) v, nfloats); +} + +/* Marshal the given value in the output buffer */ + +static void extern_rec(struct caml_extern_state* s, value v) +{ + struct extern_item * sp; + uintnat h = 0; + uintnat pos = 0; + + /* for Double_tag and Double_array_tag */ + static_assert(sizeof(double) == 8, ""); + + extern_init_position_table(s); + sp = s->extern_stack; + + while(1) { + if (Is_null(v)) { + extern_null(s); + } else if (Is_long(v)) { + extern_int(s, Long_val(v)); + } + else { + header_t hd = Hd_val(v); + tag_t tag = Tag_hd(hd); + mlsize_t sz = Wosize_hd(hd); + if (Reserved_hd(hd) != 0) { + /* Nonzero reserved header bits indicate a mixed block. */ + extern_invalid_argument(s, "output_value: mixed block"); + break; + } + + if (tag == Forward_tag) { + value f = Forward_val (v); + if (Is_block (f) + && ( Tag_val (f) == Forward_tag + || Tag_val (f) == Lazy_tag + || Tag_val (f) == Forcing_tag + /* Double_tag check because of flat float arrays */ + || Tag_val (f) == Double_tag + )){ + /* Do not short-circuit the pointer. */ + }else{ + v = f; + continue; + } + } + /* Atoms are treated specially for two reasons: they are not allocated + in the externed block, and they are automatically shared. */ + if (sz == 0) { + extern_header(s, 0, tag); + goto next_item; + } + /* Check if object already seen */ + if (extern_lookup_position(s, v, &pos, &h)) { + extern_shared_reference(s, s->obj_counter - pos); + goto next_item; + } + /* Output the contents of the object */ + switch(tag) { + case String_tag: { + mlsize_t len = caml_string_length(v); + extern_string(s, v, len); + s->size_64 += 1 + (len + 8) / 8; + extern_record_location(s, v, h); + break; + } + case Double_tag: { + extern_double(s, v); + s->size_64 += 1 + 1; + extern_record_location(s, v, h); + break; + } + case Double_array_tag: { + /* sizeof(double) == sizeof(value), per the static_assert above */ + mlsize_t nfloats = Wosize_val(v); + extern_double_array(s, v, nfloats); + s->size_64 += 1 + nfloats; + extern_record_location(s, v, h); + break; + } + case Abstract_tag: + extern_invalid_argument(s, "output_value: abstract value (Abstract)"); + break; + case Infix_tag: + /* An infix pointer into a closure block; closures cannot be + marshalled (see Closure_tag below). */ + extern_invalid_argument(s, "output_value: functional value"); + break; + case Custom_tag: + /* Custom blocks (Int32, Int64, Nativeint, Bigarray, ...) cannot + be marshalled here: their serialization functions would need the + caml_serialize_* entry points, which are not provided. */ + extern_invalid_argument(s, "output_value: abstract value (Custom)"); + break; + case Closure_tag: + /* The runtime's code-fragment table is not available here, so + code pointers, and hence closures, cannot be marshalled. */ + extern_invalid_argument(s, "output_value: functional value"); + break; + case Cont_tag: + extern_invalid_argument(s, "output_value: continuation value"); + break; + default: { + extern_header(s, sz, tag); + s->size_64 += 1 + sz; + extern_record_location(s, v, h); + /* Remember that we still have to serialize fields 1 ... sz - 1 */ + if (sz > 1) { + sp++; + if (sp >= s->extern_stack_limit) + sp = extern_resize_stack(s, sp); + sp->v = &Field(v, 1); + sp->count = sz - 1; + } + /* Continue serialization with the first field */ + v = Field(v, 0); + continue; + } + } + } + next_item: + /* Pop one more item to marshal, if any */ + if (sp == s->extern_stack) { + /* We are done. Cleanup the stack and leave the function */ + extern_free_stack(s); + extern_free_position_table(s); + return; + } + v = *((sp->v)++); + if (--(sp->count) == 0) sp--; + } + /* Never reached as function leaves with return */ +} + +static intnat extern_value(struct caml_extern_state* s, value v, + /*out*/ char header[MAX_INTEXT_HEADER_SIZE], + /*out*/ int * header_len) +{ + intnat res_len; + /* Initializations */ + s->obj_counter = 0; + s->size_64 = 0; + /* Marshal the object */ + extern_rec(s, v); + /* Record end of output */ + close_extern_output(s); + /* Write the header */ + res_len = extern_output_length(s); + if (res_len >= ((intnat)1 << 32) || s->size_64 >= ((intnat)1 << 32)) { + /* The object is too big for the small header format. + Use the big header. */ + store32(header, Intext_magic_number_big); + store32(header + 4, 0); + store64(header + 8, res_len); + store64(header + 16, s->obj_counter); + store64(header + 24, s->size_64); + *header_len = 32; + return res_len; + } + /* Use the small header format. The field at offset 12 is the size in + words when read on a 32-bit platform; 32-bit readers are not + supported, so 0 is written instead. */ + store32(header, Intext_magic_number_small); + store32(header + 4, res_len); + store32(header + 8, s->obj_counter); + store32(header + 12, 0); + store32(header + 16, s->size_64); + *header_len = 20; + return res_len; +} + +/* The entry point */ + +caml_extern_error +caml_output_value_to_malloc(value v, + /*out*/ char ** buf, + /*out*/ intnat * len) +{ + char header[MAX_INTEXT_HEADER_SIZE]; + int header_len; + intnat data_len; + char * res; + struct caml_extern_state* s = init_extern_state (); + + if (s == NULL) { + extern_error_code = CAML_EXTERN_ERROR_OUT_OF_MEMORY; + extern_error_msg = "output_value: out of memory"; + return extern_error_code; + } + extern_error_msg = NULL; + if (setjmp(s->error_return) != 0) return extern_error_code; + + init_extern_output(s); + data_len = extern_value(s, v, header, &header_len); + res = (char *) malloc(header_len + data_len); + if (res == NULL) extern_out_of_memory(s); + *buf = res; + *len = header_len + data_len; + memcpy(res, header, header_len); + res += header_len; + for (struct caml_output_block *blk = s->extern_output_first, *nextblk; + blk != NULL; + blk = nextblk) { + intnat n = blk->end - blk->data; + memcpy(res, blk->data, n); + res += n; + nextblk = blk->next; + free(blk); + } + return CAML_EXTERN_OK; +} diff --git a/extern_standalone/extern_standalone_check_big.ml b/extern_standalone/extern_standalone_check_big.ml new file mode 100644 index 0000000000000..579c6a5800a06 --- /dev/null +++ b/extern_standalone/extern_standalone_check_big.ml @@ -0,0 +1,12 @@ +(* Reads the output of extern_standalone_test_big.c back with + Marshal.from_channel and checks it against the expected list. + + Usage: ocaml extern_standalone_check_big.ml actual_big.bin *) + +let ic = open_in_bin Sys.argv.(1) + +let () = + let expected = List.init 10000 (fun i -> "item" ^ string_of_int i) in + let v : string list = Marshal.from_channel ic in + if v <> expected then (prerr_endline "big mismatch"; exit 1); + print_endline "big readback OK" diff --git a/extern_standalone/extern_standalone_check_readback.ml b/extern_standalone/extern_standalone_check_readback.ml new file mode 100644 index 0000000000000..9832504115920 --- /dev/null +++ b/extern_standalone/extern_standalone_check_readback.ml @@ -0,0 +1,43 @@ +(* Reads the output of extern_standalone_test.c back with + Marshal.from_channel and checks each value, proving the standalone + output is consumable by the real OCaml runtime. + + Usage: ocaml extern_standalone_check_readback.ml actual.bin *) + +let ic = open_in_bin Sys.argv.(1) + +type t = A of int | B of int | C of int | D of string + +let idx = ref 0 + +let check : 'a. 'a -> unit = fun expected -> + incr idx; + let v = Marshal.from_channel ic in + if v <> expected then begin + Printf.eprintf "value %d mismatch\n" !idx; + exit 1 + end + +let () = + check 42; + let pair = (1, 2) in + check pair; + check "hello"; + check (pair, pair); + check 3.14; + check [| 1.5; 2.5; 3.5 |]; + check [1; 2; 3]; + check ""; + check (-5); + check (-1000); + check 100000; + check (1 lsl 40); + check (String.make 100 'x'); + check (String.make 300 'y'); + check (D "payload"); + check ([||] : int array); + check ([1; 2; 3], "mid", 2.25); + (match input_char ic with + | exception End_of_file -> () + | _ -> prerr_endline "trailing data"; exit 1); + print_endline "readback OK" diff --git a/extern_standalone/extern_standalone_compare.py b/extern_standalone/extern_standalone_compare.py new file mode 100644 index 0000000000000..2b78eb0977d36 --- /dev/null +++ b/extern_standalone/extern_standalone_compare.py @@ -0,0 +1,31 @@ +"""Compare two files of concatenated marshal blobs (e.g. the output of +extern_standalone_test.c against that of +extern_standalone_gen_expected.ml), ignoring the header field at offset +12 of each blob (size in words when read on a 32-bit platform), which +extern_standalone.c writes as 0. + +Usage: python3 extern_standalone_compare.py actual.bin expected.bin +""" +import sys + +def blobs(path): + data = open(path, 'rb').read() + i, out = 0, [] + while i < len(data): + magic = int.from_bytes(data[i:i+4], 'big') + assert magic == 0x8495A6BE, f"unexpected magic {magic:#x} at {i}" + datalen = int.from_bytes(data[i+4:i+8], 'big') + out.append(bytearray(data[i:i+20+datalen])) + i += 20 + datalen + assert i == len(data) + return out + +a, b = blobs(sys.argv[1]), blobs(sys.argv[2]) +assert len(a) == len(b), f"blob counts differ: {len(a)} vs {len(b)}" +for n, (x, y) in enumerate(zip(a, b)): + x[12:16] = b'\0\0\0\0' + y[12:16] = b'\0\0\0\0' + if x != y: + print(f"blob {n} differs") + sys.exit(1) +print(f"OK: {len(a)} blobs identical modulo the 32-bit size field") diff --git a/extern_standalone/extern_standalone_gen_expected.ml b/extern_standalone/extern_standalone_gen_expected.ml new file mode 100644 index 0000000000000..a632cce0df94d --- /dev/null +++ b/extern_standalone/extern_standalone_gen_expected.ml @@ -0,0 +1,35 @@ +(* Generates the expected marshalled bytes for extern_standalone_test.c, + using the real Marshal module. Values must match, in order, those in + extern_standalone_test.c. + + Usage: ocaml extern_standalone_gen_expected.ml expected.bin + then compare against the test output with + extern_standalone_compare.py. *) + +let out = open_out_bin Sys.argv.(1) + +let dump v = output_string out (Marshal.to_string v []) + +(* [D _] is the fourth non-constant constructor, so it has tag 3 *) +type t = A of int | B of int | C of int | D of string + +let () = + dump 42; + let pair = (1, 2) in + dump pair; + dump "hello"; + dump (pair, pair); + dump 3.14; + dump [| 1.5; 2.5; 3.5 |]; + dump [1; 2; 3]; + dump ""; + dump (-5); + dump (-1000); + dump 100000; + dump (1 lsl 40); + dump (String.make 100 'x'); + dump (String.make 300 'y'); + dump (D "payload"); + dump [||]; + dump ([1; 2; 3], "mid", 2.25); + close_out out diff --git a/extern_standalone/extern_standalone_gen_expected_big.ml b/extern_standalone/extern_standalone_gen_expected_big.ml new file mode 100644 index 0000000000000..ac7f23dd0b476 --- /dev/null +++ b/extern_standalone/extern_standalone_gen_expected_big.ml @@ -0,0 +1,11 @@ +(* Generates the expected marshalled bytes for + extern_standalone_test_big.c, using the real Marshal module. + + Usage: ocaml extern_standalone_gen_expected_big.ml expected_big.bin *) + +let out = open_out_bin Sys.argv.(1) + +let () = + let l = List.init 10000 (fun i -> "item" ^ string_of_int i) in + output_string out (Marshal.to_string l []); + close_out out diff --git a/extern_standalone/extern_standalone_test.c b/extern_standalone/extern_standalone_test.c new file mode 100644 index 0000000000000..6e97ea629bc56 --- /dev/null +++ b/extern_standalone/extern_standalone_test.c @@ -0,0 +1,219 @@ +/* Test harness for extern_standalone.c (and for extern_standalone.cpp, + by linking this file against the C++ build instead). + + Build: cc -std=c11 -Wall -Wextra extern_standalone.c \ + extern_standalone_test.c -o test_extern + + Constructs OCaml values by hand (native-code OxCaml representation) + and marshals them, writing the concatenated results to the file given + as argv[1]; then exercises the error paths, which are self-checking. + The dumped values are intended to be compared against the output of + extern_standalone_gen_expected.ml (the same values marshalled by the + real Marshal module) using extern_standalone_compare.py: they match + byte-for-byte except for the 32-bit size field at bytes 12-15 of each + marshal header, which is written as 0 here. Reading the output back + with extern_standalone_check_readback.ml must also yield the original + values. See extern_standalone_test_big.c for a stress test. */ + +#include +#include +#include +#include + +typedef intptr_t intnat; +typedef uintptr_t uintnat; +typedef intnat value; + +typedef enum { + CAML_EXTERN_OK = 0, + CAML_EXTERN_ERROR_OUT_OF_MEMORY = 1, + CAML_EXTERN_ERROR_INVALID_ARGUMENT = 2 +} caml_extern_error; + +extern caml_extern_error caml_output_value_to_malloc(value v, char ** buf, + intnat * len); +extern const char * caml_extern_error_message(void); + +/* Value construction. Header: 8 reserved bits (zero here), 46 size bits, + 2 colour bits (NOT_MARKABLE), 8 tag bits. */ + +#define TAG_CLOSURE 247 +#define TAG_ABSTRACT 251 +#define TAG_STRING 252 +#define TAG_DOUBLE 253 +#define TAG_DOUBLE_ARRAY 254 +#define TAG_CUSTOM 255 + +static value mk_long(long n) { return ((value) n << 1) | 1; } + +static value mk_block(unsigned tag, size_t wosize) +{ + uintnat * p = malloc((wosize + 1) * sizeof(value)); + if (p == NULL) abort(); + p[0] = ((uintnat) wosize << 10) | (3u << 8) | tag; + return (value) (p + 1); +} + +static void set_field(value b, size_t i, value f) +{ + ((value *) b)[i] = f; +} + +static value mk_string(const char * str) +{ + size_t len = strlen(str); + size_t wosize = (len + sizeof(value)) / sizeof(value); + size_t bosize = wosize * sizeof(value); + value b = mk_block(TAG_STRING, wosize); + char * data = (char *) b; + memset(data, 0, bosize); + memcpy(data, str, len); + data[bosize - 1] = (char) (bosize - 1 - len); + return b; +} + +static value mk_double(double d) +{ + value b = mk_block(TAG_DOUBLE, 1); + memcpy((void *) b, &d, 8); + return b; +} + +static value mk_double_array(const double * d, size_t n) +{ + value b = mk_block(TAG_DOUBLE_ARRAY, n); + memcpy((void *) b, d, n * 8); + return b; +} + +static value mk_list(const long * elts, size_t n) +{ + value l = mk_long(0); /* [] */ + for (size_t i = n; i > 0; i--) { + value cell = mk_block(0, 2); + set_field(cell, 0, mk_long(elts[i - 1])); + set_field(cell, 1, l); + l = cell; + } + return l; +} + +static FILE * out; + +static void dump(value v) +{ + char * buf; + intnat len; + caml_extern_error err = caml_output_value_to_malloc(v, &buf, &len); + if (err != CAML_EXTERN_OK) { + fprintf(stderr, "unexpected error %d: %s\n", (int) err, + caml_extern_error_message()); + exit(2); + } + fwrite(buf, 1, len, out); + free(buf); +} + +int main(int argc, char ** argv) +{ + if (argc != 2) { fprintf(stderr, "usage: %s OUTFILE\n", argv[0]); return 2; } + out = fopen(argv[1], "wb"); + if (out == NULL) { perror("fopen"); return 2; } + + /* Same values, in the same order, as gen_expected.ml */ + dump(mk_long(42)); + + value pair = mk_block(0, 2); + set_field(pair, 0, mk_long(1)); + set_field(pair, 1, mk_long(2)); + dump(pair); + + dump(mk_string("hello")); + + value shared = mk_block(0, 2); + set_field(shared, 0, pair); + set_field(shared, 1, pair); + dump(shared); + + dump(mk_double(3.14)); + + double floats[3] = { 1.5, 2.5, 3.5 }; + dump(mk_double_array(floats, 3)); + + long ints[3] = { 1, 2, 3 }; + dump(mk_list(ints, 3)); + + dump(mk_string("")); + dump(mk_long(-5)); + dump(mk_long(-1000)); + dump(mk_long(100000)); + dump(mk_long(1L << 40)); + + char buf100[101], buf300[301]; + memset(buf100, 'x', 100); buf100[100] = '\0'; + memset(buf300, 'y', 300); buf300[300] = '\0'; + dump(mk_string(buf100)); + dump(mk_string(buf300)); + + /* A variant with a non-zero tag and an atom (empty block) */ + value variant = mk_block(3, 1); + set_field(variant, 0, mk_string("payload")); + dump(variant); + dump(mk_block(0, 0)); + + /* Nested structure exercising the extern stack */ + value nested = mk_block(0, 3); + set_field(nested, 0, mk_list(ints, 3)); + set_field(nested, 1, mk_string("mid")); + set_field(nested, 2, mk_double(2.25)); + dump(nested); + + fclose(out); + + /* Error paths, not compared against the OCaml output */ + + caml_extern_error err; + + value abstract = mk_block(TAG_ABSTRACT, 1); + set_field(abstract, 0, mk_long(0)); + char * buf; + intnat len; + err = caml_output_value_to_malloc(abstract, &buf, &len); + if (err != CAML_EXTERN_ERROR_INVALID_ARGUMENT + || strcmp(caml_extern_error_message(), + "output_value: abstract value (Abstract)") != 0) { + fprintf(stderr, "expected abstract error, got %d: %s\n", (int) err, + caml_extern_error_message()); + return 2; + } + + /* Custom blocks are now rejected */ + value custom = mk_block(TAG_CUSTOM, 2); + set_field(custom, 0, (value) &main); /* would be the ops pointer */ + set_field(custom, 1, mk_long(0)); + err = caml_output_value_to_malloc(custom, &buf, &len); + if (err != CAML_EXTERN_ERROR_INVALID_ARGUMENT + || strcmp(caml_extern_error_message(), + "output_value: abstract value (Custom)") != 0) { + fprintf(stderr, "expected custom error, got %d: %s\n", (int) err, + caml_extern_error_message()); + return 2; + } + + /* Closures are always rejected */ + value closure = mk_block(TAG_CLOSURE, 3); + set_field(closure, 0, (value) &main); + set_field(closure, 1, (value) ((1ULL << 56) + (1ULL << 55) + (2 << 1) + 1)); + set_field(closure, 2, mk_long(7)); + err = caml_output_value_to_malloc(closure, &buf, &len); + if (err != CAML_EXTERN_ERROR_INVALID_ARGUMENT + || strcmp(caml_extern_error_message(), + "output_value: functional value") != 0) { + fprintf(stderr, "expected functional-value error, got %d: %s\n", + (int) err, caml_extern_error_message()); + return 2; + } + + printf("error-path tests passed\n"); + return 0; +} diff --git a/extern_standalone/extern_standalone_test_big.c b/extern_standalone/extern_standalone_test_big.c new file mode 100644 index 0000000000000..7a01cec5cac16 --- /dev/null +++ b/extern_standalone/extern_standalone_test_big.c @@ -0,0 +1,91 @@ +/* Stress test for extern_standalone.c (or extern_standalone.cpp): + marshals a list of 10000 distinct strings, exercising position table + resize (>170 shared objects), output block growth (>8100 bytes) and + extern stack resize (list longer than 256 elements). + + Build: cc -std=c11 -Wall -Wextra extern_standalone.c \ + extern_standalone_test_big.c -o test_big + (also worth running with -fsanitize=address). Writes the marshalled + list to the file given as argv[1]; compare it against the output of + extern_standalone_gen_expected_big.ml using + extern_standalone_compare.py, and read it back with + extern_standalone_check_big.ml. */ + +#include +#include +#include +#include + +typedef intptr_t intnat; +typedef uintptr_t uintnat; +typedef intnat value; + +typedef enum { + CAML_EXTERN_OK = 0, + CAML_EXTERN_ERROR_OUT_OF_MEMORY = 1, + CAML_EXTERN_ERROR_INVALID_ARGUMENT = 2 +} caml_extern_error; + +extern caml_extern_error caml_output_value_to_malloc(value v, char ** buf, + intnat * len); +extern const char * caml_extern_error_message(void); + +static value mk_long(long n) { return ((value) n << 1) | 1; } + +static value mk_block(unsigned tag, size_t wosize) +{ + uintnat * p = malloc((wosize + 1) * sizeof(value)); + if (p == NULL) abort(); + p[0] = ((uintnat) wosize << 10) | (3u << 8) | tag; + return (value) (p + 1); +} + +static void set_field(value b, size_t i, value f) +{ + ((value *) b)[i] = f; +} + +static value mk_string(const char * str) +{ + size_t len = strlen(str); + size_t wosize = (len + sizeof(value)) / sizeof(value); + size_t bosize = wosize * sizeof(value); + value b = mk_block(252, wosize); + char * data = (char *) b; + memset(data, 0, bosize); + memcpy(data, str, len); + data[bosize - 1] = (char) (bosize - 1 - len); + return b; +} + +#define N 10000 + +int main(int argc, char ** argv) +{ + if (argc != 2) { fprintf(stderr, "usage: %s OUTFILE\n", argv[0]); return 2; } + FILE * out = fopen(argv[1], "wb"); + if (out == NULL) { perror("fopen"); return 2; } + + /* A list of N distinct strings "item0" ... "item9999" */ + value l = mk_long(0); + for (long i = N - 1; i >= 0; i--) { + char name[32]; + snprintf(name, sizeof name, "item%ld", i); + value cell = mk_block(0, 2); + set_field(cell, 0, mk_string(name)); + set_field(cell, 1, l); + l = cell; + } + + char * buf; + intnat len; + caml_extern_error err = caml_output_value_to_malloc(l, &buf, &len); + if (err != CAML_EXTERN_OK) { + fprintf(stderr, "error %d: %s\n", (int) err, caml_extern_error_message()); + return 2; + } + fwrite(buf, 1, len, out); + free(buf); + fclose(out); + return 0; +} diff --git a/extern_standalone/lldb_external_printer/README.md b/extern_standalone/lldb_external_printer/README.md new file mode 100644 index 0000000000000..cafbb02b55f26 --- /dev/null +++ b/extern_standalone/lldb_external_printer/README.md @@ -0,0 +1,82 @@ +# External pretty-printer for OxCaml values in lldb + +lldb can hand an OCaml value to a user-provided executable for +pretty-printing. The value is *marshalled out of the debugged process* by +lldb itself (`OxCamlMarshal.cpp`, an adaptation of the runtime's `extern.c` +whose memory reads go through the debugger), then piped to the executable, +which demarshals it with the ordinary `Marshal` module and prints whatever +rendering it likes. + +## Protocol + +The executable is run once per value: + +- `argv.(1)`: the OCaml type name from the debug info, with the layout + annotation stripped (`"Env.t @ value"` → `"Env.t"`). +- stdin: the value in `Marshal` wire format (read it with + `Marshal.from_channel`; open stdin in binary mode first). +- If the type is recognized: print the rendering on stdout and exit 0. + Everything written to stdout becomes the summary lldb displays. +- Otherwise: exit nonzero. lldb falls back to its built-in OCaml + formatter. stderr is discarded by lldb. + +lldb kills the printer if it takes more than 10 seconds. + +## Limitations + +Values that cannot be marshalled fall back to the built-in formatter +automatically: closures, continuations, custom blocks (`Int32.t`, `Int64.t`, +`Nativeint.t`, `Bigarray`, ...), abstract and mixed blocks, unboxed +primitives (`float#`, `int64#`, ...), values optimized into DWARF implicit +pointers, and any value whose object graph contains unreadable memory. +The marshalled output is capped (64 MiB) so that a corrupt value cannot +wedge the debugger. + +Marshal is not type-safe: the printer must be linked against exactly the +type definitions the debuggee uses, and must only demarshal at the type +named in `argv.(1)`. + +## Building the example + +```sh +./build.sh # uses /Users/mark/dev/oxcaml2/_install/bin/ocamlopt +``` + +This produces `test_program` (the debuggee) and `printer` (the external +pretty-printer, which recognizes `Test_types.point` and `Test_types.shape`). + +## Using it + +The recommended route is the OxCaml plugin setting, which keeps the +built-in formatter as a fallback: + +``` +(lldb) settings set plugin.oxcaml.display.external-summary-executable \ + /path/to/lldb_external_printer/printer +(lldb) b Test_program.process_point +(lldb) run +(lldb) frame variable +``` + +Put the `settings set` line in `~/.lldbinit` to make it permanent. + +To see what lldb sends the printer (including the exact type name passed as +`argv.(1)` — adjust `printer.ml`'s matching to taste): + +``` +(lldb) log enable oxcaml formatting +``` + +There is also a generic, language-independent mechanism: + +``` +(lldb) type summary add ocaml_value --summary-executable /path/to/printer +``` + +This registers the executable through lldb's ordinary type-summary +machinery (it works for any language; for OCaml values the input is the +marshalled value, for other languages the value's raw bytes). Note that a +summary added this way lives in the "default" category, which takes +precedence over the OxCaml plugin's category — so if the printer rejects a +type, lldb shows the plain value rather than falling back to the built-in +OCaml formatter. Prefer the plugin setting for everyday use. diff --git a/extern_standalone/lldb_external_printer/build.sh b/extern_standalone/lldb_external_printer/build.sh new file mode 100755 index 0000000000000..f377ddd8923c8 --- /dev/null +++ b/extern_standalone/lldb_external_printer/build.sh @@ -0,0 +1,11 @@ +#!/bin/sh +# Build the example debuggee and the external pretty-printer. +set -e +OCAMLOPT=${OCAMLOPT:-/Users/mark/dev/oxcaml2/_install/bin/ocamlopt} +cd "$(dirname "$0")" + +"$OCAMLOPT" -g -c test_types.ml +"$OCAMLOPT" -g -o test_program test_types.cmx test_program.ml +"$OCAMLOPT" -o printer test_types.cmx printer.ml + +echo "Built: test_program (debuggee) and printer (external pretty-printer)" diff --git a/extern_standalone/lldb_external_printer/printer b/extern_standalone/lldb_external_printer/printer new file mode 100755 index 0000000000000..2b3b102016665 Binary files /dev/null and b/extern_standalone/lldb_external_printer/printer differ diff --git a/extern_standalone/lldb_external_printer/printer.cmi b/extern_standalone/lldb_external_printer/printer.cmi new file mode 100644 index 0000000000000..aa5c3be85b276 Binary files /dev/null and b/extern_standalone/lldb_external_printer/printer.cmi differ diff --git a/extern_standalone/lldb_external_printer/printer.cmx b/extern_standalone/lldb_external_printer/printer.cmx new file mode 100644 index 0000000000000..a4ad4bc3b6abc Binary files /dev/null and b/extern_standalone/lldb_external_printer/printer.cmx differ diff --git a/extern_standalone/lldb_external_printer/printer.ml b/extern_standalone/lldb_external_printer/printer.ml new file mode 100644 index 0000000000000..3f243217f839c --- /dev/null +++ b/extern_standalone/lldb_external_printer/printer.ml @@ -0,0 +1,50 @@ +(* Example external pretty-printer for lldb's OxCaml support. + + Protocol (see lldb's OxCamlExternalPrinter.cpp): + - argv.(1) is the OCaml type name as recorded in the debug info, with the + layout annotation stripped (e.g. "Test_types.point"). + - The value arrives on stdin in OCaml's Marshal wire format. + - If the type is recognized, print a rendering of the value on stdout and + exit with code 0. The whole of stdout becomes the summary lldb shows. + - Otherwise exit with a nonzero code; lldb then falls back to its + built-in OCaml formatter. + + CAUTION: Marshal.from_channel is not type-safe. Only read the value at + the type the debugger reported, and keep this printer linked against the + same type definitions as the program being debugged. + + The exact spelling of the type name depends on the DWARF the compiler + emitted. To see what lldb passes, run in lldb: + log enable oxcaml formatting + and look for the "external printer: running ..." line. The [matches] + helper below is deliberately forgiving while experimenting. *) + +let matches type_name candidates = + List.exists + (fun candidate -> + String.equal type_name candidate + || (* Accept module-qualified spellings, e.g. "Test_types.point" for + candidate "point", and stamped spellings such as "point/123". *) + (let l = String.length type_name and lc = String.length candidate in + (l > lc && String.sub type_name (l - lc - 1) (lc + 1) = "." ^ candidate) + || (l > lc + && String.sub type_name 0 (lc + 1) = candidate ^ "/")) + ) + candidates + +let () = + if Array.length Sys.argv < 2 then exit 2; + let type_name = Sys.argv.(1) in + set_binary_mode_in stdin true; + if matches type_name [ "point"; "Test_types.point" ] then begin + let (p : Test_types.point) = Marshal.from_channel stdin in + print_string (Test_types.string_of_point p) + end + else if matches type_name [ "shape"; "Test_types.shape" ] then begin + let (s : Test_types.shape) = Marshal.from_channel stdin in + print_string (Test_types.string_of_shape s) + end + else + (* Not a type this printer knows: tell lldb to use its built-in + formatter. *) + exit 1 diff --git a/extern_standalone/lldb_external_printer/printer.o b/extern_standalone/lldb_external_printer/printer.o new file mode 100644 index 0000000000000..604ce00aabd17 Binary files /dev/null and b/extern_standalone/lldb_external_printer/printer.o differ diff --git a/extern_standalone/lldb_external_printer/test_program b/extern_standalone/lldb_external_printer/test_program new file mode 100755 index 0000000000000..4cca0b62a869b Binary files /dev/null and b/extern_standalone/lldb_external_printer/test_program differ diff --git a/extern_standalone/lldb_external_printer/test_program.cmi b/extern_standalone/lldb_external_printer/test_program.cmi new file mode 100644 index 0000000000000..d0b47b9de0365 Binary files /dev/null and b/extern_standalone/lldb_external_printer/test_program.cmi differ diff --git a/extern_standalone/lldb_external_printer/test_program.cmx b/extern_standalone/lldb_external_printer/test_program.cmx new file mode 100644 index 0000000000000..49a295a265df0 Binary files /dev/null and b/extern_standalone/lldb_external_printer/test_program.cmx differ diff --git a/extern_standalone/lldb_external_printer/test_program.ml b/extern_standalone/lldb_external_printer/test_program.ml new file mode 100644 index 0000000000000..40c3ec94cb429 --- /dev/null +++ b/extern_standalone/lldb_external_printer/test_program.ml @@ -0,0 +1,22 @@ +(* Debuggee for exercising the external pretty-printer. Break on + Test_program.process_point or Test_program.process_shape and look at the + parameters with "frame variable". *) + +open Test_types + +let process_point (p : point) = + Printf.printf "processing %s\n%!" (string_of_point p); + p.x + p.y + +let process_shape (s : shape) = + match s with + | Circle (c, r) -> c.x + c.y + r + | Line (a, b) -> a.x + b.x + +let () = + let p = { x = 3; y = 4 } in + let n = process_point p in + let c = Circle (p, 10) in + let l = Line (p, { x = 7; y = 1 }) in + let m = process_shape c + process_shape l in + Printf.printf "results: %d %d\n" n m diff --git a/extern_standalone/lldb_external_printer/test_program.o b/extern_standalone/lldb_external_printer/test_program.o new file mode 100644 index 0000000000000..26ddbc5bfea51 Binary files /dev/null and b/extern_standalone/lldb_external_printer/test_program.o differ diff --git a/extern_standalone/lldb_external_printer/test_types.cmi b/extern_standalone/lldb_external_printer/test_types.cmi new file mode 100644 index 0000000000000..3570e658ad134 Binary files /dev/null and b/extern_standalone/lldb_external_printer/test_types.cmi differ diff --git a/extern_standalone/lldb_external_printer/test_types.cmx b/extern_standalone/lldb_external_printer/test_types.cmx new file mode 100644 index 0000000000000..db86666610144 Binary files /dev/null and b/extern_standalone/lldb_external_printer/test_types.cmx differ diff --git a/extern_standalone/lldb_external_printer/test_types.ml b/extern_standalone/lldb_external_printer/test_types.ml new file mode 100644 index 0000000000000..31aa725f8ed07 --- /dev/null +++ b/extern_standalone/lldb_external_printer/test_types.ml @@ -0,0 +1,19 @@ +(* Types shared between the debugged program (test_program.ml) and the + external pretty-printer (printer.ml). The printer must be linked against + the same type definitions it demarshals at, since Marshal is not + type-safe. *) + +type point = { x : int; y : int } + +type shape = + | Circle of point * int + | Line of point * point + +let string_of_point (p : point) = Printf.sprintf "(%d, %d)" p.x p.y + +let string_of_shape (s : shape) = + match s with + | Circle (c, r) -> + Printf.sprintf "circle centred on %s with radius %d" (string_of_point c) r + | Line (a, b) -> + Printf.sprintf "line from %s to %s" (string_of_point a) (string_of_point b) diff --git a/extern_standalone/lldb_external_printer/test_types.o b/extern_standalone/lldb_external_printer/test_types.o new file mode 100644 index 0000000000000..d4c248ef1cc2e Binary files /dev/null and b/extern_standalone/lldb_external_printer/test_types.o differ diff --git a/lldb/include/lldb/Core/PluginManager.h b/lldb/include/lldb/Core/PluginManager.h index 5499e99025d8a..7761c47be093f 100644 --- a/lldb/include/lldb/Core/PluginManager.h +++ b/lldb/include/lldb/Core/PluginManager.h @@ -692,6 +692,14 @@ class PluginManager { Debugger &debugger, const lldb::OptionValuePropertiesSP &properties_sp, llvm::StringRef description, bool is_global_property); + static lldb::OptionValuePropertiesSP + GetSettingForOxCamlLanguagePlugin(Debugger &debugger, + llvm::StringRef setting_name); + + static bool CreateSettingForOxCamlLanguagePlugin( + Debugger &debugger, const lldb::OptionValuePropertiesSP &properties_sp, + llvm::StringRef description, bool is_global_property); + // // Plugin Info+Enable Declarations // diff --git a/lldb/include/lldb/Core/Value.h b/lldb/include/lldb/Core/Value.h index 3714621b469ec..fd34a82cfbb67 100644 --- a/lldb/include/lldb/Core/Value.h +++ b/lldb/include/lldb/Core/Value.h @@ -18,7 +18,9 @@ #include "lldb/lldb-private-types.h" #include "llvm/ADT/APInt.h" +#include "llvm/ADT/ArrayRef.h" +#include #include #include @@ -26,6 +28,7 @@ namespace lldb_private { class DataExtractor; +class DWARFExpressionDelegate; class ExecutionContext; class Module; class Stream; @@ -49,7 +52,46 @@ class Value { LoadAddress, /// A host address value (for memory in the process that < A is /// using liblldb). - HostAddress + HostAddress, + /// A pointer that does not exist in the process being debugged: the + /// pointer itself was optimized away, but the value it would point at + /// is described by another DWARF DIE (DW_OP_implicit_pointer). m_value + /// is unset (deliberately, so that type-unaware consumers asking for a + /// number receive their fail value rather than something that could be + /// mistaken for a real pointer); the pointee is identified by the + /// ImplicitPointerInfo and can be materialized on demand with + /// DWARFExpression::DereferenceImplicitPointer(). + ImplicitPointer + }; + + /// Identifies the pointed-at value of an implicit pointer (a Value whose + /// type is ValueType::ImplicitPointer). + struct ImplicitPointerInfo { + /// The DWARF unit that contained the DW_OP_implicit_pointer expression; + /// used to resolve die_offset. The pointer is owned by the module's + /// symbol file, which must outlive this value (the same lifetime + /// assumption Value already makes for m_context). + const DWARFExpressionDelegate *delegate = nullptr; + /// The offset of the DIE describing the pointed-at value, relative to + /// the start of the .debug_info section. + uint64_t die_offset = 0; + /// Byte offset into the pointed-at value. + int64_t byte_offset = 0; + }; + + /// A byte range within a composite (DW_OP_piece) value's buffer whose + /// piece is an implicit pointer. The buffer bytes in the range are + /// meaningless placeholders: the table entry, never the bytes, is the + /// piece's value. Consumers reading pointer-sized quantities out of a + /// composite buffer must consult the table (see + /// FindImplicitPointerPiece) before interpreting the bytes. + struct ImplicitPointerPiece { + /// Byte offset of the piece within the value's buffer. + uint64_t buffer_offset = 0; + /// Size of the piece in bytes. + uint64_t byte_size = 0; + /// The pointed-at value. + ImplicitPointerInfo pointee; }; /// Type that describes Value::m_context. @@ -82,6 +124,43 @@ class Value { ValueType GetValueType() const; + /// Make this value an implicit pointer (ValueType::ImplicitPointer) to the + /// value described by the DIE at \a die_offset in the .debug_info section + /// of \a delegate's symbol file, at \a byte_offset bytes into that value. + /// Clears the scalar and data buffer; the pointer has no numeric value. + void SetImplicitPointer(const DWARFExpressionDelegate *delegate, + uint64_t die_offset, int64_t byte_offset); + + /// The pointee of this value when GetValueType() == + /// ValueType::ImplicitPointer, std::nullopt otherwise. + const std::optional &GetImplicitPointer() const { + return m_implicit_pointer; + } + + /// Record that the byte range [buffer_offset, buffer_offset + byte_size) + /// of this value's buffer is an implicit-pointer piece whose actual value + /// is \a pointee. + void AddImplicitPointerPiece(uint64_t buffer_offset, uint64_t byte_size, + const ImplicitPointerInfo &pointee) { + m_implicit_pointer_pieces.push_back({buffer_offset, byte_size, pointee}); + } + + /// The implicit-pointer pieces of this composite value, if any. + llvm::ArrayRef GetImplicitPointerPieces() const { + return m_implicit_pointer_pieces; + } + + /// Return the implicit-pointer piece exactly covering + /// [buffer_offset, buffer_offset + byte_size), or nullptr if there is + /// none. + const ImplicitPointerPiece * + FindImplicitPointerPiece(uint64_t buffer_offset, uint64_t byte_size) const { + for (const ImplicitPointerPiece &piece : m_implicit_pointer_pieces) + if (piece.buffer_offset == buffer_offset && piece.byte_size == byte_size) + return &piece; + return nullptr; + } + AddressType GetValueAddressType() const; ContextType GetContextType() const { return m_context_type; } @@ -182,6 +261,11 @@ class Value { ValueType m_value_type = ValueType::Scalar; ContextType m_context_type = ContextType::Invalid; DataBufferHeap m_data_buffer; + /// Only meaningful when m_value_type == ValueType::ImplicitPointer. + std::optional m_implicit_pointer; + /// Byte ranges of m_data_buffer that are implicit-pointer pieces of a + /// composite (DW_OP_piece) value. + std::vector m_implicit_pointer_pieces; }; class ValueList { diff --git a/lldb/include/lldb/DataFormatters/TypeSummary.h b/lldb/include/lldb/DataFormatters/TypeSummary.h index 589f68c2ce314..a6489248f8c04 100644 --- a/lldb/include/lldb/DataFormatters/TypeSummary.h +++ b/lldb/include/lldb/DataFormatters/TypeSummary.h @@ -21,6 +21,8 @@ #include "lldb/Core/FormatEntity.h" #include "lldb/Utility/Status.h" #include "lldb/Utility/StructuredData.h" +#include "llvm/ADT/ArrayRef.h" +#include "llvm/Support/Error.h" namespace llvm { class MemoryBuffer; @@ -48,7 +50,14 @@ class TypeSummaryOptions { class TypeSummaryImpl { public: - enum class Kind { eSummaryString, eScript, eBytecode, eCallback, eInternal }; + enum class Kind { + eSummaryString, + eScript, + eBytecode, + eCallback, + eInternal, + eExternal + }; virtual ~TypeSummaryImpl() = default; @@ -423,6 +432,56 @@ struct ScriptSummaryFormat : public TypeSummaryImpl { const ScriptSummaryFormat &operator=(const ScriptSummaryFormat &) = delete; }; +/// A summary produced by running an external program supplied by the user. +/// +/// The program is run once per value to be summarized. It receives the +/// value's type name as its only command-line argument and, on its standard +/// input, a language-defined serialization of the value (see +/// Language::GetExternalFormatterInput; by default the value's raw bytes, +/// for OCaml the value in Marshal format). If the program recognizes the +/// type it must print the summary text on its standard output and exit with +/// code 0. Any other exit code, or a failure to run the program at all, +/// makes the summary fail so that display falls back to LLDB's default +/// rendering of the value. +class ExternalSummaryFormat : public TypeSummaryImpl { + std::string m_executable_path; + +public: + ExternalSummaryFormat(const TypeSummaryImpl::Flags &flags, + llvm::StringRef executable_path); + + ~ExternalSummaryFormat() override = default; + + const char *GetExecutablePath() const { return m_executable_path.c_str(); } + + bool FormatObject(ValueObject *valobj, std::string &dest, + const TypeSummaryOptions &options) override; + + /// Run \p executable_path with \p type_name as its only argument, sending + /// \p input on its standard input, and return its standard output (with + /// trailing newlines removed). An error is returned if the program cannot + /// be run, does not finish in time, or exits with a nonzero code. + static llvm::Expected + RunSummaryExecutable(llvm::StringRef executable_path, + llvm::StringRef type_name, + llvm::ArrayRef input); + + std::string GetDescription() override; + + std::string GetName() override; + + static bool classof(const TypeSummaryImpl *S) { + return S->GetKind() == Kind::eExternal; + } + + typedef std::shared_ptr SharedPointer; + +private: + ExternalSummaryFormat(const ExternalSummaryFormat &) = delete; + const ExternalSummaryFormat & + operator=(const ExternalSummaryFormat &) = delete; +}; + /// A summary formatter that is defined in LLDB formmater bytecode. class BytecodeSummaryFormat : public TypeSummaryImpl { std::unique_ptr m_bytecode; diff --git a/lldb/include/lldb/Expression/DWARFExpression.h b/lldb/include/lldb/Expression/DWARFExpression.h index 37853c0b5a8fc..f762568c7b603 100644 --- a/lldb/include/lldb/Expression/DWARFExpression.h +++ b/lldb/include/lldb/Expression/DWARFExpression.h @@ -19,9 +19,149 @@ #include "llvm/DebugInfo/DWARF/DWARFLocationExpression.h" #include "llvm/Support/Error.h" #include +#include +#include namespace lldb_private { +class CallEdge; + +/// Context for resolving a DW_OP_entry_value that refers to the entry of a +/// function which is not represented by a live stack frame. +/// +/// Normally DW_OP_entry_value is resolved by walking the live call stack from +/// the frame being evaluated up to its caller, then finding the call site that +/// produced the activation (see Evaluate_DW_OP_entry_value). That walk is +/// impossible while the stack frame list is still being built, which is when +/// tail-call frames are synthesized: the function whose entry value is +/// referenced was reached through the static call graph and has no frame of +/// its own to walk back from, and re-entering the frame list would deadlock. +/// +/// This structure supplies that information explicitly instead. \ref edge is +/// the call that produced the activation whose entry is referenced; its call +/// site parameters map a callee-side location (matched against the +/// DW_OP_entry_value sub-expression) to a caller-side location. That +/// caller-side location is then evaluated either in \ref caller_frame, when +/// the caller is a live frame, or — when the caller is itself a frameless link +/// in a tail-call chain — by recursively resolving any DW_OP_entry_value it +/// contains against \ref outer, chaining back through the tail-call chain until +/// a live frame is reached. +/// +/// A frameless caller can only contribute values that are themselves further +/// entry values or constants. If its caller-side location needs state that a +/// vanished activation cannot provide (a register, frame base, or stack slot), +/// evaluation fails rather than producing an unreliable value. +struct EntryValueResolutionContext { + /// The call edge whose call site parameters describe the entry value. Never + /// null in a well-formed context. + const CallEdge *edge = nullptr; + + /// The live frame of the caller (the function containing \ref edge's call + /// site), or null when that caller is itself a frameless link in a tail-call + /// chain. + StackFrame *caller_frame = nullptr; + + /// When \ref caller_frame is null, how to resolve a DW_OP_entry_value that + /// appears within the caller-side location expression. A null \ref outer + /// together with a null \ref caller_frame means the chain cannot be resolved. + const EntryValueResolutionContext *outer = nullptr; +}; + +/// \class DWARFExpressionDelegate DWARFExpression.h +/// "lldb/Expression/DWARFExpression.h" Interface through which the DWARF +/// expression evaluator accesses the DWARF unit that contains the expression +/// being evaluated. Implemented by the DWARF symbol file plugin's DWARFUnit. +/// +/// This is a top-level class (rather than nested in DWARFExpression) so that +/// it can be forward declared by code that may not include Expression +/// headers, such as lldb_private::Value. +class DWARFExpressionDelegate { +public: + DWARFExpressionDelegate() = default; + virtual ~DWARFExpressionDelegate() = default; + + virtual uint16_t GetVersion() const = 0; + virtual dw_addr_t GetBaseAddress() const = 0; + virtual uint8_t GetAddressByteSize() const = 0; + + /// Return the size in bytes of an offset in the DWARF format of this + /// unit: 4 bytes for the 32-bit DWARF format, 8 bytes for the 64-bit + /// DWARF format. This is the size of the operand of DW_OP_call_ref and + /// of the DIE reference operand of DW_OP_implicit_pointer. + virtual uint8_t GetDWARFOffsetByteSize() const = 0; + + virtual llvm::Expected> + GetDIEBitSizeAndSign(uint64_t relative_die_offset) const = 0; + + /// Return the DW_AT_location expression of the DIE referenced by a + /// DW_OP_call2, DW_OP_call4, DW_OP_call_ref or DW_OP_implicit_pointer + /// operation, together with the delegate of the unit that contains the + /// expression (which may not be this unit when the offset is not unit + /// relative). + /// + /// \param[in] die_offset + /// The offset of the referenced DIE. If \a unit_relative is true, + /// the offset is relative to the start of this unit + /// (DW_OP_call2/DW_OP_call4), otherwise it is an offset in the + /// .debug_info section (DW_OP_call_ref, DW_OP_implicit_pointer). + /// + /// \param[in] pc_function_offset + /// The offset of the program counter from the entry point of the + /// function the frame is executing in, or std::nullopt when there is + /// no frame or function. When the referenced DIE's DW_AT_location is + /// a location list rather than a single expression, this selects the + /// list entry that applies. A function-relative offset is used + /// because location list ranges are expressed in the address space + /// of the DWARF that defines them, which is not necessarily the + /// address space the program runs in (for example on Darwin without + /// a dSYM, where the DWARF lives in the .o files); the offset is the + /// same in both spaces and is re-anchored at the DWARF-side + /// function's entry point. + /// + /// \return + /// The location expression data and the delegate to evaluate it + /// with. An empty (zero length) DataExtractor is returned when the + /// referenced DIE exists but has no location description that + /// applies: either it has no DW_AT_location attribute at all, or the + /// attribute is a location list none of whose entries cover the + /// program counter. The caller decides what that means (for the + /// DW_OP_call operations the call has no effect, which + /// compiler-emitted guard expressions rely on to detect unavailable + /// values; for DW_OP_implicit_pointer the pointed-at value is + /// unavailable). An error is returned if the DIE cannot be resolved, + /// or if its DW_AT_location is a location list and + /// \a pc_function_offset is unknown (the applicable entry then + /// cannot be determined at all). + virtual llvm::Expected< + std::pair> + GetDIELocationExpression(uint64_t die_offset, bool unit_relative, + std::optional pc_function_offset) + const = 0; + + /// Return the value of the DW_AT_const_value attribute of the DIE at + /// \a die_offset in the .debug_info section. This is used when + /// dereferencing a DW_OP_implicit_pointer whose referenced DIE describes + /// the pointed-at value with a constant rather than a location. Block and + /// string forms are returned as host byte buffers, integer forms as + /// scalars. Returns std::nullopt when the DIE exists but has no + /// DW_AT_const_value attribute, and an error when the DIE cannot be + /// resolved. + virtual llvm::Expected> + GetDIEConstValue(uint64_t die_offset) const = 0; + + virtual dw_addr_t ReadAddressFromDebugAddrSection(uint32_t index) const = 0; + virtual lldb::offset_t + GetVendorDWARFOpcodeSize(const DataExtractor &data, + const lldb::offset_t data_offset, + const uint8_t op) const = 0; + virtual bool ParseVendorDWARFOpcode(uint8_t op, const DataExtractor &opcodes, + lldb::offset_t &offset, + std::vector &stack) const = 0; + + DWARFExpressionDelegate(const DWARFExpressionDelegate &) = delete; + DWARFExpressionDelegate &operator=(const DWARFExpressionDelegate &) = delete; +}; + /// \class DWARFExpression DWARFExpression.h /// "lldb/Expression/DWARFExpression.h" Encapsulates a DWARF location /// expression and interprets it. @@ -37,29 +177,9 @@ class DWARFExpression { public: using Stack = std::vector; - class Delegate { - public: - Delegate() = default; - virtual ~Delegate() = default; - - virtual uint16_t GetVersion() const = 0; - virtual dw_addr_t GetBaseAddress() const = 0; - virtual uint8_t GetAddressByteSize() const = 0; - virtual llvm::Expected> - GetDIEBitSizeAndSign(uint64_t relative_die_offset) const = 0; - virtual dw_addr_t ReadAddressFromDebugAddrSection(uint32_t index) const = 0; - virtual lldb::offset_t - GetVendorDWARFOpcodeSize(const DataExtractor &data, - const lldb::offset_t data_offset, - const uint8_t op) const = 0; - virtual bool ParseVendorDWARFOpcode(uint8_t op, - const DataExtractor &opcodes, - lldb::offset_t &offset, - Stack &stack) const = 0; - - Delegate(const Delegate &) = delete; - Delegate &operator=(const Delegate &) = delete; - }; + /// Compatibility alias; the delegate predates its hoisting to a top-level + /// class and is widely referred to as DWARFExpression::Delegate. + using Delegate = DWARFExpressionDelegate; DWARFExpression(); @@ -148,11 +268,41 @@ class DWARFExpression { /// \return /// True on success; false otherwise. If error_ptr is non-NULL, /// details of the failure are provided through it. + /// \param[in] entry_value_ctx + /// Optional explicit context for resolving a DW_OP_entry_value whose + /// referenced function has no live stack frame (used while synthesizing + /// tail-call frames). When null, DW_OP_entry_value is resolved by walking + /// the live call stack instead. \see EntryValueResolutionContext. static llvm::Expected Evaluate(ExecutionContext *exe_ctx, RegisterContext *reg_ctx, lldb::ModuleSP module_sp, const DataExtractor &opcodes, const Delegate *dwarf_cu, const lldb::RegisterKind reg_set, - const Value *initial_value_ptr, const Value *object_address_ptr); + const Value *initial_value_ptr, const Value *object_address_ptr, + const EntryValueResolutionContext *entry_value_ctx = nullptr); + + /// Materialize the value an implicit pointer points at. + /// + /// \a implicit_pointer must be a Value of type + /// Value::ValueType::ImplicitPointer, i.e. the result of evaluating a + /// DWARF expression that ends in DW_OP_implicit_pointer: a pointer that + /// does not exist in the process being debugged, but whose pointed-at + /// value is described by another DIE. This function evaluates the + /// DW_AT_location of that DIE in the given context and applies the + /// implicit pointer's byte offset, yielding the pointed-at value. The + /// result can be any kind of Value: a load address (the pointee does + /// exist in the process's memory), a host address or scalar (the pointee + /// only exists in the debugger), or another implicit pointer (the pointee + /// is itself an eliminated pointer); in the latter cases the value lives + /// in LLDB's own address space, never the process's. + /// + /// Note that failure is always reported as an error: a pointee that is + /// unavailable (e.g. the referenced DIE has no DW_AT_location) is never + /// conflated with a pointee that evaluates to the value zero. + static llvm::Expected + DereferenceImplicitPointer(const Value &implicit_pointer, + ExecutionContext *exe_ctx, + RegisterContext *reg_ctx, + lldb::ModuleSP module_sp); bool GetExpressionData(DataExtractor &data) const { data = m_data; diff --git a/lldb/include/lldb/Expression/DWARFExpressionList.h b/lldb/include/lldb/Expression/DWARFExpressionList.h index d303ad834b354..76e81cc210b7b 100644 --- a/lldb/include/lldb/Expression/DWARFExpressionList.h +++ b/lldb/include/lldb/Expression/DWARFExpressionList.h @@ -93,6 +93,23 @@ class DWARFExpressionList { std::function const &link_address_callback); + /// Rewrite the operand of the DW_OP_addr operation in each expression of + /// the list by passing it through \a link_address_callback. This is used + /// when the expressions come from a .o file behind a debug map (Darwin + /// without a dSYM): DW_OP_addr operands then hold .o file addresses, + /// which must be linked into the executable's address space to be + /// resolvable at evaluation time. Expressions without a DW_OP_addr are + /// left untouched. + /// + /// \return + /// True if every DW_OP_addr operand found was successfully linked + /// and rewritten; false if any was left unlinked (the callback + /// returned LLDB_INVALID_ADDRESS, e.g. the referenced object did + /// not make it into the linked executable) or could not be parsed. + bool LinkDWOPAddrOperands( + std::function const + &link_address_callback); + bool MatchesOperand(StackFrame &frame, const Instruction::Operand &operand) const; @@ -120,11 +137,16 @@ class DWARFExpressionList { void SetModule(const lldb::ModuleSP &module) { m_module_wp = module; } - llvm::Expected Evaluate(ExecutionContext *exe_ctx, - RegisterContext *reg_ctx, - lldb::addr_t func_load_addr, - const Value *initial_value_ptr, - const Value *object_address_ptr) const; + /// Return the module whose DWARF the expressions came from, if it is + /// still loaded. Addresses pushed by DW_OP_addr within the expressions + /// are file addresses in this module's address space. + lldb::ModuleSP GetModule() const { return m_module_wp.lock(); } + + llvm::Expected + Evaluate(ExecutionContext *exe_ctx, RegisterContext *reg_ctx, + lldb::addr_t func_load_addr, const Value *initial_value_ptr, + const Value *object_address_ptr, + const EntryValueResolutionContext *entry_value_ctx = nullptr) const; private: // RangeDataVector requires a comparator for DWARFExpression, but it doesn't diff --git a/lldb/include/lldb/Symbol/Function.h b/lldb/include/lldb/Symbol/Function.h index 21b3f9ab4a70c..306fce320a3ca 100644 --- a/lldb/include/lldb/Symbol/Function.h +++ b/lldb/include/lldb/Symbol/Function.h @@ -274,8 +274,16 @@ class CallEdge { /// /// Note that this might lazily invoke the DWARF parser. A register context /// from the caller's activation is needed to find indirect call targets. - virtual Function *GetCallee(ModuleList &images, - ExecutionContext &exe_ctx) = 0; + /// + /// \param[in] entry_value_ctx + /// Optional context for resolving a DW_OP_entry_value in an indirect + /// call target whose containing function has no live frame, as happens + /// while synthesizing tail-call frames. Null in the common case, where + /// DW_OP_entry_value resolves against the live call stack. + /// \see EntryValueResolutionContext. + virtual Function * + GetCallee(ModuleList &images, ExecutionContext &exe_ctx, + const EntryValueResolutionContext *entry_value_ctx = nullptr) = 0; /// Get the load PC address of the instruction which executes after the call /// returns. Returns LLDB_INVALID_ADDRESS iff this is a tail call. \p caller @@ -339,7 +347,9 @@ class DirectCallEdge : public CallEdge { lldb::addr_t caller_address, bool is_tail_call, CallSiteParameterArray &¶meters); - Function *GetCallee(ModuleList &images, ExecutionContext &exe_ctx) override; + Function * + GetCallee(ModuleList &images, ExecutionContext &exe_ctx, + const EntryValueResolutionContext *entry_value_ctx) override; private: void ParseSymbolFileAndResolve(ModuleList &images); @@ -368,7 +378,9 @@ class IndirectCallEdge : public CallEdge { AddrType caller_address_type, lldb::addr_t caller_address, bool is_tail_call, CallSiteParameterArray &¶meters); - Function *GetCallee(ModuleList &images, ExecutionContext &exe_ctx) override; + Function * + GetCallee(ModuleList &images, ExecutionContext &exe_ctx, + const EntryValueResolutionContext *entry_value_ctx) override; private: // Used to describe an indirect call. diff --git a/lldb/include/lldb/Target/Language.h b/lldb/include/lldb/Target/Language.h index d3af1cf8edbb5..f415c4acff078 100644 --- a/lldb/include/lldb/Target/Language.h +++ b/lldb/include/lldb/Target/Language.h @@ -355,6 +355,26 @@ class Language : public PluginInterface { // it wants LLDB to honor it should return an appropriate closure here virtual DumpValueObjectOptions::DeclPrintingHelper GetDeclPrintingHelper(); + /// Produce the input for an external summary program (see + /// ExternalSummaryFormat) applied to \p valobj. + /// + /// Returns the type name to pass to the program as its argument and stores + /// the bytes to send on the program's standard input into \p data. The + /// default implementation sends the value's raw bytes together with its + /// display type name; language plugins can override this to send a + /// language-specific serialization instead (the OCaml plugin, for example, + /// sends the value in Marshal format). + virtual llvm::Expected + GetExternalFormatterInput(ValueObject &valobj, std::vector &data) { + return GetDefaultExternalFormatterInput(valobj, data); + } + + /// The behaviour of the base implementation of GetExternalFormatterInput, + /// also usable when a value has no language plugin at all. + static llvm::Expected + GetDefaultExternalFormatterInput(ValueObject &valobj, + std::vector &data); + virtual LazyBool IsLogicalTrue(ValueObject &valobj, Status &error); // for a ValueObject of some "reference type", if the value points to the diff --git a/lldb/source/API/SBTypeSummary.cpp b/lldb/source/API/SBTypeSummary.cpp index 58ec068ab9600..45226cabb01b4 100644 --- a/lldb/source/API/SBTypeSummary.cpp +++ b/lldb/source/API/SBTypeSummary.cpp @@ -372,6 +372,9 @@ bool SBTypeSummary::IsEqualTo(lldb::SBTypeSummary &rhs) { return GetOptions() == rhs.GetOptions(); case TypeSummaryImpl::Kind::eInternal: return (m_opaque_sp.get() == rhs.m_opaque_sp.get()); + case TypeSummaryImpl::Kind::eExternal: + return m_opaque_sp->GetName() == rhs.m_opaque_sp->GetName() && + GetOptions() == rhs.GetOptions(); } return false; diff --git a/lldb/source/Commands/CommandObjectType.cpp b/lldb/source/Commands/CommandObjectType.cpp index 19cd3ff2972e9..ae07f09ece3c1 100644 --- a/lldb/source/Commands/CommandObjectType.cpp +++ b/lldb/source/Commands/CommandObjectType.cpp @@ -13,6 +13,7 @@ #include "lldb/DataFormatters/DataVisualization.h" #include "lldb/DataFormatters/FormatClasses.h" #include "lldb/Host/Config.h" +#include "lldb/Host/FileSystem.h" #include "lldb/Host/OptionParser.h" #include "lldb/Host/StreamFile.h" #include "lldb/Interpreter/CommandInterpreter.h" @@ -147,6 +148,7 @@ class CommandObjectTypeSummaryAdd : public CommandObjectParsed, std::string m_python_function; bool m_is_add_script = false; std::string m_category; + std::string m_external_executable; uint32_t m_ptr_match_depth = 1; }; @@ -158,6 +160,8 @@ class CommandObjectTypeSummaryAdd : public CommandObjectParsed, bool Execute_StringSummary(Args &command, CommandReturnObject &result); + bool Execute_ExternalSummary(Args &command, CommandReturnObject &result); + public: CommandObjectTypeSummaryAdd(CommandInterpreter &interpreter); @@ -1218,6 +1222,9 @@ Status CommandObjectTypeSummaryAdd::CommandOptions::SetOptionValue( case 'P': m_is_add_script = true; break; + case '\x02': + m_external_executable = std::string(option_arg); + break; case 'w': m_category = std::string(option_arg); break; @@ -1246,6 +1253,7 @@ void CommandObjectTypeSummaryAdd::CommandOptions::OptionParsingStarting( m_format_string = ""; m_is_add_script = false; m_category = "default"; + m_external_executable = ""; } #if LLDB_ENABLE_PYTHON @@ -1365,6 +1373,56 @@ bool CommandObjectTypeSummaryAdd::Execute_ScriptSummary( #endif +bool CommandObjectTypeSummaryAdd::Execute_ExternalSummary( + Args &command, CommandReturnObject &result) { + const size_t argc = command.GetArgumentCount(); + + if (argc < 1 && !m_options.m_name) { + result.AppendErrorWithFormat("%s takes one or more args.\n", + m_cmd_name.c_str()); + return false; + } + + FileSpec exe_spec(m_options.m_external_executable); + FileSystem::Instance().Resolve(exe_spec); + if (!FileSystem::Instance().Exists(exe_spec)) + result.AppendWarningWithFormat( + "the executable \"%s\" does not currently exist - " + "the summary will fail until it does.\n", + exe_spec.GetPath().c_str()); + + TypeSummaryImplSP entry = std::make_shared( + m_options.m_flags, exe_spec.GetPath()); + + Status error; + for (auto &arg_entry : command.entries()) { + if (arg_entry.ref().empty()) { + result.AppendError("empty typenames not allowed"); + return false; + } + + AddSummary(ConstString(arg_entry.ref()), entry, m_options.m_match_type, + m_options.m_category, &error); + + if (error.Fail()) { + result.AppendError(error.AsCString()); + return false; + } + } + + if (m_options.m_name) { + AddNamedSummary(m_options.m_name, entry, &error); + if (error.Fail()) { + result.AppendError(error.AsCString()); + result.AppendError("added to types, but not given a name"); + return false; + } + } + + result.SetStatus(eReturnStatusSuccessFinishNoResult); + return result.Succeeded(); +} + bool CommandObjectTypeSummaryAdd::Execute_StringSummary( Args &command, CommandReturnObject &result) { const size_t argc = command.GetArgumentCount(); @@ -1551,6 +1609,11 @@ void CommandObjectTypeSummaryAdd::DoExecute(Args &command, return; } + if (!m_options.m_external_executable.empty()) { + Execute_ExternalSummary(command, result); + return; + } + Execute_StringSummary(command, result); } diff --git a/lldb/source/Commands/Options.td b/lldb/source/Commands/Options.td index acb741081cac3..775b1d8eb839d 100644 --- a/lldb/source/Commands/Options.td +++ b/lldb/source/Commands/Options.td @@ -1292,11 +1292,22 @@ let Command = "type summary add" in { Desc<"Give the name of a Python function to use for this type.">; def type_summary_add_input_python : Option<"input-python", "P">, Group<3>, Desc<"Input Python code to use for this type manually.">; - def type_summary_add_expand : Option<"expand", "e">, Groups<[2,3]>, + def type_summary_add_summary_executable : + Option<"summary-executable", "\\x02">, Group<4>, Arg<"Filename">, + Desc<"Path to an executable that produces the summary. The executable is " + "run with the value's type name as its only argument and receives a " + "serialization of the value on its standard input (the value's raw bytes " + "by default; language plug-ins may define a richer encoding, e.g. OCaml " + "values are sent in Marshal format). If it recognizes the type it must " + "print the summary on its standard output and exit with code 0; any " + "other exit code makes LLDB fall back to the default display for the " + "value.">; + def type_summary_add_expand : Option<"expand", "e">, Groups<[2,3,4]>, Desc<"Expand aggregate data types to show children on separate lines.">; - def type_summary_add_hide_empty : Option<"hide-empty", "h">, Groups<[2,3]>, + def type_summary_add_hide_empty : Option<"hide-empty", "h">, Groups<[2,3,4]>, Desc<"Do not expand aggregate data types with no children.">; - def type_summary_add_name : Option<"name", "n">, Groups<[2,3]>, Arg<"Name">, + def type_summary_add_name : Option<"name", "n">, Groups<[2,3,4]>, + Arg<"Name">, Desc<"A name for this summary string.">; } diff --git a/lldb/source/Core/PluginManager.cpp b/lldb/source/Core/PluginManager.cpp index 3f20a96edc187..3c9cc2953b4ab 100644 --- a/lldb/source/Core/PluginManager.cpp +++ b/lldb/source/Core/PluginManager.cpp @@ -2071,6 +2071,7 @@ static constexpr llvm::StringLiteral kJITLoaderPluginName("jit-loader"); static constexpr llvm::StringLiteral kStructuredDataPluginName("structured-data"); static constexpr llvm::StringLiteral kCPlusPlusLanguagePlugin("cplusplus"); +static constexpr llvm::StringLiteral kOxCamlLanguagePlugin("oxcaml"); lldb::OptionValuePropertiesSP PluginManager::GetSettingForDynamicLoaderPlugin(Debugger &debugger, @@ -2243,6 +2244,20 @@ bool PluginManager::CreateSettingForCPlusPlusLanguagePlugin( properties_sp, description, is_global_property); } +lldb::OptionValuePropertiesSP +PluginManager::GetSettingForOxCamlLanguagePlugin(Debugger &debugger, + llvm::StringRef setting_name) { + return GetSettingForPlugin(debugger, setting_name, kOxCamlLanguagePlugin); +} + +bool PluginManager::CreateSettingForOxCamlLanguagePlugin( + Debugger &debugger, const lldb::OptionValuePropertiesSP &properties_sp, + llvm::StringRef description, bool is_global_property) { + return CreateSettingForPlugin(debugger, kOxCamlLanguagePlugin, + "Settings for OxCaml language plug-ins", + properties_sp, description, is_global_property); +} + // // Plugin Info+Enable Implementations // diff --git a/lldb/source/Core/Value.cpp b/lldb/source/Core/Value.cpp index c91b3f852f986..567932898ed9d 100644 --- a/lldb/source/Core/Value.cpp +++ b/lldb/source/Core/Value.cpp @@ -55,7 +55,9 @@ Value::Value(const void *bytes, int len) Value::Value(const Value &v) : m_value(v.m_value), m_compiler_type(v.m_compiler_type), m_context(v.m_context), m_value_type(v.m_value_type), - m_context_type(v.m_context_type), m_data_buffer() { + m_context_type(v.m_context_type), m_data_buffer(), + m_implicit_pointer(v.m_implicit_pointer), + m_implicit_pointer_pieces(v.m_implicit_pointer_pieces) { const uintptr_t rhs_value = (uintptr_t)v.m_value.ULongLong(LLDB_INVALID_ADDRESS); if ((rhs_value != 0) && @@ -74,6 +76,8 @@ Value &Value::operator=(const Value &rhs) { m_context = rhs.m_context; m_value_type = rhs.m_value_type; m_context_type = rhs.m_context_type; + m_implicit_pointer = rhs.m_implicit_pointer; + m_implicit_pointer_pieces = rhs.m_implicit_pointer_pieces; const uintptr_t rhs_value = (uintptr_t)rhs.m_value.ULongLong(LLDB_INVALID_ADDRESS); if ((rhs_value != 0) && @@ -110,10 +114,24 @@ void Value::Dump(Stream *strm) { Value::ValueType Value::GetValueType() const { return m_value_type; } +void Value::SetImplicitPointer(const DWARFExpressionDelegate *delegate, + uint64_t die_offset, int64_t byte_offset) { + // Deliberately leave the scalar cleared: an implicit pointer has no + // numeric value, and consumers that ask for one without checking the value + // type should receive their fail value, not something that could be + // mistaken for a real pointer (such as zero). + m_value.Clear(); + m_data_buffer.Clear(); + m_implicit_pointer = ImplicitPointerInfo{delegate, die_offset, byte_offset}; + m_implicit_pointer_pieces.clear(); + m_value_type = ValueType::ImplicitPointer; +} + AddressType Value::GetValueAddressType() const { switch (m_value_type) { case ValueType::Invalid: case ValueType::Scalar: + case ValueType::ImplicitPointer: break; case ValueType::LoadAddress: return eAddressTypeLoad; @@ -159,6 +177,9 @@ size_t Value::AppendDataToHostBuffer(const Value &rhs) { Status error; switch (rhs.GetValueType()) { case ValueType::Invalid: + case ValueType::ImplicitPointer: + // An implicit pointer has no byte representation; do not append + // anything (in particular, not zeros). return 0; case ValueType::Scalar: { const size_t scalar_size = rhs.m_value.GetByteSize(); @@ -300,6 +321,7 @@ lldb::Format Value::GetValueDefaultFormat() { bool Value::GetData(DataExtractor &data) { switch (m_value_type) { case ValueType::Invalid: + case ValueType::ImplicitPointer: return false; case ValueType::Scalar: if (m_value.GetData(data)) @@ -340,6 +362,13 @@ Status Value::GetValueAsData(ExecutionContext *exe_ctx, DataExtractor &data, case ValueType::Invalid: error = Status::FromErrorString("invalid value"); break; + case ValueType::ImplicitPointer: + // The pointer itself has no memory representation; only the value it + // points at can be materialized, via + // DWARFExpression::DereferenceImplicitPointer(). + error = Status::FromErrorString( + "value is an implicit pointer whose numeric value is not available"); + break; case ValueType::Scalar: { data.SetByteOrder(endian::InlHostByteOrder()); if (ast_type.IsValid()) @@ -593,6 +622,7 @@ Scalar &Value::ResolveValue(ExecutionContext *exe_ctx, Module *module) { switch (m_value_type) { case ValueType::Invalid: case ValueType::Scalar: // raw scalar value + case ValueType::ImplicitPointer: // no numeric value to resolve break; case ValueType::FileAddress: @@ -641,6 +671,8 @@ void Value::Clear() { m_context = nullptr; m_context_type = ContextType::Invalid; m_data_buffer.Clear(); + m_implicit_pointer.reset(); + m_implicit_pointer_pieces.clear(); } const char *Value::GetValueTypeAsCString(ValueType value_type) { @@ -655,6 +687,8 @@ const char *Value::GetValueTypeAsCString(ValueType value_type) { return "load address"; case ValueType::HostAddress: return "host address"; + case ValueType::ImplicitPointer: + return "implicit pointer"; }; llvm_unreachable("enum cases exhausted."); } diff --git a/lldb/source/DataFormatters/TypeSummary.cpp b/lldb/source/DataFormatters/TypeSummary.cpp index 6aa290698cd12..c70986c071cf0 100644 --- a/lldb/source/DataFormatters/TypeSummary.cpp +++ b/lldb/source/DataFormatters/TypeSummary.cpp @@ -16,10 +16,18 @@ #include "lldb/DataFormatters/ValueObjectPrinter.h" #include "lldb/Interpreter/CommandInterpreter.h" #include "lldb/Symbol/CompilerType.h" +#include "lldb/Target/Language.h" #include "lldb/Target/StackFrame.h" #include "lldb/Target/Target.h" +#include "lldb/Utility/LLDBLog.h" +#include "lldb/Utility/Log.h" #include "lldb/Utility/StreamString.h" #include "lldb/ValueObject/ValueObject.h" +#include "llvm/Support/FileSystem.h" +#include "llvm/Support/FileUtilities.h" +#include "llvm/Support/MemoryBuffer.h" +#include "llvm/Support/Program.h" +#include "llvm/Support/raw_ostream.h" using namespace lldb; using namespace lldb_private; @@ -59,6 +67,8 @@ std::string TypeSummaryImpl::GetSummaryKindName() { return "c++"; case Kind::eBytecode: return "bytecode"; + case Kind::eExternal: + return "external"; } llvm_unreachable("Unknown type kind name"); } @@ -241,6 +251,129 @@ std::string ScriptSummaryFormat::GetDescription() { std::string ScriptSummaryFormat::GetName() { return m_script_formatter_name; } +ExternalSummaryFormat::ExternalSummaryFormat( + const TypeSummaryImpl::Flags &flags, llvm::StringRef executable_path) + : TypeSummaryImpl(Kind::eExternal, flags), + m_executable_path(executable_path.str()) {} + +llvm::Expected +ExternalSummaryFormat::RunSummaryExecutable(llvm::StringRef executable_path, + llvm::StringRef type_name, + llvm::ArrayRef input) { + // The input reaches the program through a temporary file redirected to its + // standard input; from the program's point of view this is the same as a + // pipe, but avoids any risk of deadlock on large inputs. + llvm::SmallString<128> input_path; + int input_fd; + if (std::error_code ec = llvm::sys::fs::createTemporaryFile( + "lldb-external-summary-in", "bin", input_fd, input_path)) + return llvm::createStringError( + ec, "cannot create temporary file for external summary input"); + llvm::FileRemover input_remover(input_path); + { + llvm::raw_fd_ostream input_stream(input_fd, /*shouldClose=*/true); + input_stream.write(reinterpret_cast(input.data()), + input.size()); + input_stream.flush(); + if (input_stream.has_error()) + return llvm::createStringError( + input_stream.error(), + "cannot write temporary file for external summary input"); + } + + llvm::SmallString<128> output_path; + if (std::error_code ec = llvm::sys::fs::createTemporaryFile( + "lldb-external-summary-out", "txt", output_path)) + return llvm::createStringError( + ec, "cannot create temporary file for external summary output"); + llvm::FileRemover output_remover(output_path); + + // An empty redirect sends the program's stderr to /dev/null so that its + // diagnostics cannot corrupt LLDB's terminal. + std::optional redirects[3] = { + input_path.str(), output_path.str(), llvm::StringRef("")}; + + // Guard against a wedged program: a summary is produced interactively, so + // a few seconds is already generous. + constexpr unsigned timeout_in_seconds = 10; + + std::string error_msg; + bool execution_failed = false; + int exit_code = llvm::sys::ExecuteAndWait( + executable_path, {executable_path, type_name}, /*Env=*/std::nullopt, + redirects, timeout_in_seconds, /*MemoryLimit=*/0, &error_msg, + &execution_failed); + if (execution_failed) + return llvm::createStringError("cannot run '%s': %s", + executable_path.str().c_str(), + error_msg.c_str()); + if (exit_code != 0) + return llvm::createStringError( + "'%s' did not recognize type '%s' (exit code %d)", + executable_path.str().c_str(), type_name.str().c_str(), exit_code); + + llvm::ErrorOr> output_buffer = + llvm::MemoryBuffer::getFile(output_path); + if (!output_buffer) + return llvm::createStringError( + output_buffer.getError(), "cannot read external summary output file"); + + llvm::StringRef output = (*output_buffer)->getBuffer().rtrim("\r\n"); + return output.str(); +} + +bool ExternalSummaryFormat::FormatObject(ValueObject *valobj, + std::string &dest, + const TypeSummaryOptions &options) { + dest.clear(); + if (!valobj || m_executable_path.empty()) + return false; + + Log *log = GetLog(LLDBLog::DataFormatters); + + // Ask the value's language plugin for the serialized form to send to the + // program (and the type name to pass it); without a language plugin, send + // the value's raw bytes. + Language *language = Language::FindPlugin(options.GetLanguage()); + if (!language) + language = Language::FindPlugin(valobj->GetObjectRuntimeLanguage()); + + std::vector data; + llvm::Expected type_name = + language ? language->GetExternalFormatterInput(*valobj, data) + : Language::GetDefaultExternalFormatterInput(*valobj, data); + if (!type_name) { + LLDB_LOG_ERROR(log, type_name.takeError(), + "external summary: cannot serialize value: {0}"); + return false; + } + + llvm::Expected output = + RunSummaryExecutable(m_executable_path, *type_name, data); + if (!output) { + LLDB_LOG_ERROR(log, output.takeError(), "external summary: {0}"); + return false; + } + + dest = std::move(*output); + return true; +} + +std::string ExternalSummaryFormat::GetDescription() { + StreamString sstr; + sstr.Printf("`%s`%s%s%s%s%s%s ptr-match-depth=%u", + m_executable_path.c_str(), Cascades() ? "" : " (not cascading)", + !DoesPrintChildren(nullptr) ? "" : " (show children)", + !DoesPrintValue(nullptr) ? " (hide value)" : "", + SkipsPointers() ? " (skip pointers)" : "", + SkipsReferences() ? " (skip references)" : "", + HideNames(nullptr) ? " (hide member names)" : "", + GetPtrMatchDepth()); + return std::string(sstr.GetString()); +} + +std::string ExternalSummaryFormat::GetName() { return m_executable_path; } + BytecodeSummaryFormat::BytecodeSummaryFormat( const TypeSummaryImpl::Flags &flags, std::unique_ptr bytecode) diff --git a/lldb/source/Expression/DWARFExpression.cpp b/lldb/source/Expression/DWARFExpression.cpp index 52891fcefd68b..8ec9f8c0aeb71 100644 --- a/lldb/source/Expression/DWARFExpression.cpp +++ b/lldb/source/Expression/DWARFExpression.cpp @@ -156,13 +156,20 @@ GetOpcodeDataSize(const DataExtractor &data, const lldb::offset_t data_offset, case DW_OP_APPLE_uninit: case DW_OP_PGI_omp_thread_num: case DW_OP_hi_user: - case DW_OP_GNU_implicit_pointer: break; case DW_OP_addr: - case DW_OP_call_ref: // 0x9a 1 address sized offset of DIE (DWARF3) return data.GetAddressByteSize(); + // DW_OP_call_ref takes a single operand: the offset of a DIE in the + // .debug_info section. Its size is that of an offset in the DWARF format + // of the compile unit: 4 bytes for the 32-bit DWARF format, 8 bytes for + // the 64-bit DWARF format. Fall back to the address size if the compile + // unit is not available. + case DW_OP_call_ref: // 0x9a DWARF offset sized offset of DIE (DWARF3) + return dwarf_cu ? dwarf_cu->GetDWARFOffsetByteSize() + : data.GetAddressByteSize(); + // Opcodes with no arguments case DW_OP_deref: // 0x06 case DW_OP_dup: // 0x12 @@ -356,12 +363,20 @@ GetOpcodeDataSize(const DataExtractor &data, const lldb::offset_t data_offset, return offset - data_offset; } - case DW_OP_implicit_pointer: // 0xa0 4-byte (or 8-byte for DWARF 64) constant - // + LEB128 + case DW_OP_implicit_pointer: // 0xa0 DWARF offset sized DIE offset + // + SLEB128 (DWARF5) + case DW_OP_GNU_implicit_pointer: // 0xf2 GNU extension equivalent { data.Skip_LEB128(&offset); - return (dwarf_cu ? dwarf_cu->GetAddressByteSize() : 4) + offset - - data_offset; + lldb::offset_t ref_size = 4; + if (dwarf_cu) { + ref_size = dwarf_cu->GetDWARFOffsetByteSize(); + // The GNU extension used an address sized reference in pre-DWARF 3 + // units, which had no notion of an offset sized reference. + if (op == DW_OP_GNU_implicit_pointer && dwarf_cu->GetVersion() < 3) + ref_size = dwarf_cu->GetAddressByteSize(); + } + return ref_size + offset - data_offset; } case DW_OP_GNU_entry_value: @@ -589,12 +604,12 @@ bool DWARFExpression::LinkThreadLocalStorage( return true; } -static llvm::Error Evaluate_DW_OP_entry_value(DWARFExpression::Stack &stack, - ExecutionContext *exe_ctx, - RegisterContext *reg_ctx, - const DataExtractor &opcodes, - lldb::offset_t &opcode_offset, - Log *log) { +static llvm::Error +Evaluate_DW_OP_entry_value(DWARFExpression::Stack &stack, + ExecutionContext *exe_ctx, RegisterContext *reg_ctx, + const DataExtractor &opcodes, + lldb::offset_t &opcode_offset, Log *log, + const EntryValueResolutionContext *entry_value_ctx) { // DW_OP_entry_value(sub-expr) describes the location a variable had upon // function entry: this variable location is presumed to be optimized out at // the current PC value. The caller of the function may have call site @@ -640,84 +655,152 @@ static llvm::Error Evaluate_DW_OP_entry_value(DWARFExpression::Stack &stack, // and evaluates the corresponding location for that parameter in `parent`. // 1. Find the function which pushed the current frame onto the stack. - if ((!exe_ctx || !exe_ctx->HasTargetScope()) || !reg_ctx) { - return llvm::createStringError("no exe/reg context"); - } - - StackFrame *current_frame = exe_ctx->GetFramePtr(); - Thread *thread = exe_ctx->GetThreadPtr(); - if (!current_frame || !thread) - return llvm::createStringError("no current frame/thread"); + if (!exe_ctx || !exe_ctx->HasTargetScope()) + return llvm::createStringError("no exe/target context"); + // A register context is only needed for the live-stack walk below; an + // explicit resolution context carries everything it needs itself. + if (!entry_value_ctx && !reg_ctx) + return llvm::createStringError("no register context"); Target &target = exe_ctx->GetTargetRef(); - StackFrameSP parent_frame = nullptr; - addr_t return_pc = LLDB_INVALID_ADDRESS; - uint32_t current_frame_idx = current_frame->GetFrameIndex(); - - for (uint32_t parent_frame_idx = current_frame_idx + 1;; parent_frame_idx++) { - parent_frame = thread->GetStackFrameAtIndex(parent_frame_idx); - // If this is null, we're at the end of the stack. - if (!parent_frame) - break; - - // Record the first valid return address, even if this is an inlined frame, - // in order to look up the associated call edge in the first non-inlined - // parent frame. - if (return_pc == LLDB_INVALID_ADDRESS) { - return_pc = parent_frame->GetFrameCodeAddress().GetLoadAddress(&target); - LLDB_LOG(log, "immediate ancestor with pc = {0:x}", return_pc); - } - - // If we've found an inlined frame, skip it (these have no call site - // parameters). - if (parent_frame->IsInlined()) - continue; + ModuleList &modlist = target.GetImages(); - // We've found the first non-inlined parent frame. - break; - } - if (!parent_frame || !parent_frame->GetRegisterContext()) { - return llvm::createStringError("no parent frame with reg ctx"); - } + // The call edge whose call site parameters describe the entry value, plus how + // to evaluate the matched caller-side location: either in a live frame + // (\ref caller_frame), or — for a frameless link in a tail-call chain — by + // chaining any DW_OP_entry_value it contains through \ref outer_ctx. + const CallEdge *call_edge = nullptr; + StackFrameSP caller_frame; + const EntryValueResolutionContext *outer_ctx = nullptr; + // Backing storage for an [outer_ctx] synthesized in the live-stack-walk path + // below, when the current activation was entered by a tail call from a + // frameless function that the (live) parent frame called. Kept at function + // scope so [outer_ctx] remains valid through the parameter evaluation. + std::optional tail_link_ctx; + + if (entry_value_ctx) { + // Explicit resolution: the referenced function has no live frame (this + // happens while synthesizing tail-call frames). Use the supplied edge and + // caller context directly. Walking the live stack here is both impossible + // (the function has no frame) and unsafe (it would re-enter the stack frame + // list while it is being built). + call_edge = entry_value_ctx->edge; + if (!call_edge) + return llvm::createStringError("entry value context has no call edge"); + if (entry_value_ctx->caller_frame) + caller_frame = entry_value_ctx->caller_frame->shared_from_this(); + outer_ctx = entry_value_ctx->outer; + if (!caller_frame && !outer_ctx) + return llvm::createStringError( + "entry value caller is frameless and cannot be resolved"); + } else { + StackFrame *current_frame = exe_ctx->GetFramePtr(); + Thread *thread = exe_ctx->GetThreadPtr(); + if (!current_frame || !thread) + return llvm::createStringError("no current frame/thread"); + + addr_t return_pc = LLDB_INVALID_ADDRESS; + uint32_t current_frame_idx = current_frame->GetFrameIndex(); + + for (uint32_t parent_frame_idx = current_frame_idx + 1;; + parent_frame_idx++) { + caller_frame = thread->GetStackFrameAtIndex(parent_frame_idx); + // If this is null, we're at the end of the stack. + if (!caller_frame) + break; - Function *parent_func = - parent_frame->GetSymbolContext(eSymbolContextFunction).function; - if (!parent_func) - return llvm::createStringError("no parent function"); + // Record the first valid return address, even if this is an inlined + // frame, in order to look up the associated call edge in the first + // non-inlined parent frame. + if (return_pc == LLDB_INVALID_ADDRESS) { + return_pc = caller_frame->GetFrameCodeAddress().GetLoadAddress(&target); + LLDB_LOG(log, "immediate ancestor with pc = {0:x}", return_pc); + } - // 2. Find the call edge in the parent function responsible for creating the - // current activation. - Function *current_func = - current_frame->GetSymbolContext(eSymbolContextFunction).function; - if (!current_func) - return llvm::createStringError("no current function"); + // If we've found an inlined frame, skip it (these have no call site + // parameters). + if (caller_frame->IsInlined()) + continue; - CallEdge *call_edge = nullptr; - ModuleList &modlist = target.GetImages(); - ExecutionContext parent_exe_ctx = *exe_ctx; - parent_exe_ctx.SetFrameSP(parent_frame); - if (!parent_frame->IsArtificial()) { - // If the parent frame is not artificial, the current activation may be - // produced by an ambiguous tail call. In this case, refuse to proceed. - call_edge = parent_func->GetCallEdgeForReturnAddress(return_pc, target); - if (!call_edge) { - return llvm::createStringError( - llvm::formatv("no call edge for retn-pc = {0:x} in parent frame {1}", - return_pc, parent_func->GetName())); - } - Function *callee_func = call_edge->GetCallee(modlist, parent_exe_ctx); - if (callee_func != current_func) { - return llvm::createStringError( - "ambiguous call sequence, can't find real parent frame"); + // We've found the first non-inlined parent frame. + break; } - } else { - // The StackFrameList solver machinery has deduced that an unambiguous tail - // call sequence that produced the current activation. The first edge in - // the parent that points to the current function must be valid. - for (auto &edge : parent_func->GetTailCallingEdges()) { - if (edge->GetCallee(modlist, parent_exe_ctx) == current_func) { - call_edge = edge.get(); - break; + if (!caller_frame || !caller_frame->GetRegisterContext()) + return llvm::createStringError("no parent frame with reg ctx"); + + Function *parent_func = + caller_frame->GetSymbolContext(eSymbolContextFunction).function; + if (!parent_func) + return llvm::createStringError("no parent function"); + + // 2. Find the call edge in the parent function responsible for creating + // the current activation. + Function *current_func = + current_frame->GetSymbolContext(eSymbolContextFunction).function; + if (!current_func) + return llvm::createStringError("no current function"); + + ExecutionContext parent_exe_ctx = *exe_ctx; + parent_exe_ctx.SetFrameSP(caller_frame); + if (!caller_frame->IsArtificial()) { + // The parent frame is not artificial, so the current activation was + // produced either by a direct call from the parent, or by a tail call + // from a frameless function that the parent called. + CallEdge *return_edge = + parent_func->GetCallEdgeForReturnAddress(return_pc, target); + if (!return_edge) + return llvm::createStringError(llvm::formatv( + "no call edge for retn-pc = {0:x} in parent frame {1}", return_pc, + parent_func->GetName())); + Function *callee_func = return_edge->GetCallee( + modlist, parent_exe_ctx, /*entry_value_ctx=*/nullptr); + if (callee_func == current_func) { + // The parent called the current function directly. + call_edge = return_edge; + } else if (callee_func) { + // The parent called \ref callee_func, which is not the current function + // but may have entered it by a tail call, leaving no frame of its own + // (e.g. a thunk or a partial-application veneer that loads its arguments + // and tail-calls the real callee). Find the unique tail-calling edge + // from \ref callee_func to the current function and resolve the entry + // value through it. That edge is frameless: \ref callee_func's own entry + // values, and any indirect callee it names, chain back through \ref + // return_edge to this live parent frame. + tail_link_ctx = EntryValueResolutionContext{ + return_edge, caller_frame.get(), /*outer=*/nullptr}; + for (auto &edge : callee_func->GetTailCallingEdges()) { + if (edge->GetCallee(modlist, parent_exe_ctx, &*tail_link_ctx) != + current_func) + continue; + if (call_edge) { + // More than one tail call reaches the current function: ambiguous. + call_edge = nullptr; + break; + } + call_edge = edge.get(); + } + if (!call_edge) + return llvm::createStringError( + "ambiguous call sequence, can't find real parent frame"); + // \ref callee_func has no live frame; evaluate the matched parameter as + // a frameless tail-call link chained to the parent via \ref return_edge. + caller_frame = nullptr; + outer_ctx = &*tail_link_ctx; + } else { + // The parent's indirect call target could not be resolved. + return llvm::createStringError( + "ambiguous call sequence, can't find real parent frame"); + } + } else { + // The StackFrameList solver machinery has deduced that an unambiguous + // tail call sequence produced the current activation. The first edge in + // the parent that points to the current function must be valid. + for (auto &edge : parent_func->GetTailCallingEdges()) { + if (edge->GetCallee(modlist, parent_exe_ctx, + /*entry_value_ctx=*/nullptr) == current_func) { + call_edge = edge.get(); + break; + } } } } @@ -764,17 +847,49 @@ static llvm::Error Evaluate_DW_OP_entry_value(DWARFExpression::Stack &stack, // subexpresion whenever llvm does. const DWARFExpressionList ¶m_expr = matched_param->LocationInCaller; - llvm::Expected maybe_result = param_expr.Evaluate( - &parent_exe_ctx, parent_frame->GetRegisterContext().get(), - LLDB_INVALID_ADDRESS, - /*initial_value_ptr=*/nullptr, - /*object_address_ptr=*/nullptr); + // Evaluate the caller-side location to obtain the entry value. + llvm::Expected maybe_result = + [&]() -> llvm::Expected { + if (caller_frame) { + // The caller is a live frame: evaluate the location in it as usual. + ExecutionContext caller_exe_ctx = *exe_ctx; + caller_exe_ctx.SetFrameSP(caller_frame); + return param_expr.Evaluate(&caller_exe_ctx, + caller_frame->GetRegisterContext().get(), + LLDB_INVALID_ADDRESS, + /*initial_value_ptr=*/nullptr, + /*object_address_ptr=*/nullptr); + } + // The caller is itself a frameless link in a tail-call chain. Its + // registers and stack are gone; only further entry values and constants + // can be recovered. Evaluate with no frame and no register context so that + // any access to the vanished activation's state fails cleanly, and chain a + // nested DW_OP_entry_value to the outer resolution context. + ExecutionContext frameless_ctx(exe_ctx->GetTargetSP(), + /*get_process=*/true); + return param_expr.Evaluate(&frameless_ctx, /*reg_ctx=*/nullptr, + LLDB_INVALID_ADDRESS, + /*initial_value_ptr=*/nullptr, + /*object_address_ptr=*/nullptr, outer_ctx); + }(); if (!maybe_result) { LLDB_LOG(log, "Evaluate_DW_OP_entry_value: call site param evaluation failed"); return maybe_result.takeError(); } + // A DW_AT_call_value expression produces a value, not a location, so a + // DW_OP_addr within it (e.g. a pointer argument to a statically + // allocated object) denotes the address as the value. Resolve such a + // file address here, where the module it belongs to (the caller + // expression's own) is known: the caller may be in a different module + // than the callee whose expression we resume below, and under a Darwin + // debug map each .o has its own file address space, so resolving it any + // later could consult the wrong module. + if (maybe_result->GetValueType() == Value::ValueType::FileAddress) + maybe_result->ConvertToLoadAddress(param_expr.GetModule().get(), + exe_ctx->GetTargetPtr()); + stack.push_back(*maybe_result); return llvm::Error::success(); } @@ -913,6 +1028,12 @@ static llvm::Error Evaluate_DW_OP_deref(DWARFExpression::Stack &stack, stack.back().GetScalar() = pointer_value; stack.back().ClearContext(); } break; + case Value::ValueType::ImplicitPointer: + // Implicit pointers can only be dereferenced by materializing the + // pointed-at value via its DIE (see + // DWARFExpression::DereferenceImplicitPointer), not by reading memory. + return llvm::createStringError( + "DW_OP_deref of an implicit pointer is not supported"); case Value::ValueType::Invalid: return llvm::createStringError("invalid value type for DW_OP_deref"); } @@ -940,19 +1061,67 @@ static Scalar DerefSizeExtractDataHelper(uint8_t *addr_bytes, return addr_data.GetAddress(&addr_data_offset); } -llvm::Expected DWARFExpression::Evaluate( +/// The maximum number of nested DW_OP_call2, DW_OP_call4 and DW_OP_call_ref +/// operations that may be active at once. This guards against cyclic DIE +/// references in (malformed) DWARF, which would otherwise recurse forever. +static constexpr uint32_t g_max_dwarf_call_depth = 64; + +/// Return the offset of the program counter from the entry point of the +/// function the frame is executing in, for selecting the entry of a +/// location list that applies at that point in the program (see +/// DWARFExpressionDelegate::GetDIELocationExpression). Returns std::nullopt +/// when there is no frame or no function, never an offset that could be +/// mistaken for a real one. +static std::optional +GetPCFunctionOffsetForLocationLists(ExecutionContext *exe_ctx, + RegisterContext *reg_ctx) { + StackFrame *frame = exe_ctx ? exe_ctx->GetFramePtr() : nullptr; + Address pc; + if (!reg_ctx || !reg_ctx->GetPCForSymbolication(pc)) { + if (!frame) + return std::nullopt; + lldb::RegisterContextSP reg_ctx_sp = frame->GetRegisterContext(); + if (!reg_ctx_sp || !reg_ctx_sp->GetPCForSymbolication(pc)) + return std::nullopt; + } + if (!pc.IsValid() || !frame) + return std::nullopt; + const SymbolContext &sc = + frame->GetSymbolContext(lldb::eSymbolContextFunction); + if (!sc.function) + return std::nullopt; + Target *target = exe_ctx->GetTargetPtr(); + lldb::addr_t pc_addr = pc.GetLoadAddress(target); + lldb::addr_t func_addr = sc.function->GetAddress().GetLoadAddress(target); + if (pc_addr == LLDB_INVALID_ADDRESS || func_addr == LLDB_INVALID_ADDRESS) { + // Not loaded into a process (e.g. static analysis of a binary): both + // addresses are still comparable as file addresses. + pc_addr = pc.GetFileAddress(); + func_addr = sc.function->GetAddress().GetFileAddress(); + } + if (pc_addr == LLDB_INVALID_ADDRESS || func_addr == LLDB_INVALID_ADDRESS || + pc_addr < func_addr) + return std::nullopt; + return pc_addr - func_addr; +} + +/// Evaluate the DWARF expression in \a opcodes against \a stack. +/// +/// This implements the opcode interpreter loop of DWARFExpression::Evaluate, +/// which provides and initializes the evaluation state and derives the +/// result. It is split out so that DW_OP_call2, DW_OP_call4 and +/// DW_OP_call_ref can evaluate the DW_AT_location expression of the +/// referenced DIE against the caller's state, as if the called expression's +/// opcodes appeared in place of the call operation. +static llvm::Error EvaluateOpcodes( ExecutionContext *exe_ctx, RegisterContext *reg_ctx, lldb::ModuleSP module_sp, const DataExtractor &opcodes, const DWARFExpression::Delegate *dwarf_cu, - const lldb::RegisterKind reg_kind, const Value *initial_value_ptr, - const Value *object_address_ptr) { - - if (opcodes.GetByteSize() == 0) - return llvm::createStringError( - "no location, value may have been optimized out"); - - Stack stack; - + const lldb::RegisterKind reg_kind, const Value *object_address_ptr, + DWARFExpression::Stack &stack, + LocationDescriptionKind &dwarf4_location_description_kind, Value &pieces, + uint64_t &op_piece_offset, const uint32_t call_depth, + const EntryValueResolutionContext *entry_value_ctx) { Process *process = nullptr; StackFrame *frame = nullptr; Target *target = nullptr; @@ -965,17 +1134,10 @@ llvm::Expected DWARFExpression::Evaluate( if (reg_ctx == nullptr && frame) reg_ctx = frame->GetRegisterContext().get(); - if (initial_value_ptr) - stack.push_back(*initial_value_ptr); - lldb::offset_t offset = 0; Value tmp; uint32_t reg_num; - /// Insertion point for evaluating multi-piece expression. - uint64_t op_piece_offset = 0; - Value pieces; // Used for DW_OP_piece - Log *log = GetLog(LLDBLog::Expressions); // A generic type is "an integral type that has the size of an address and an // unspecified signedness". For now, just use the signedness of the operand. @@ -990,11 +1152,6 @@ llvm::Expected DWARFExpression::Evaluate( !is_signed)); }; - // The default kind is a memory location. This is updated by any - // operation that changes this, such as DW_OP_stack_value, and reset - // by composition operations like DW_OP_piece. - LocationDescriptionKind dwarf4_location_description_kind = Memory; - while (opcodes.ValidOffset(offset)) { const lldb::offset_t op_offset = offset; const uint8_t op = opcodes.GetU8(&offset); @@ -1218,6 +1375,10 @@ llvm::Expected DWARFExpression::Evaluate( } break; + case Value::ValueType::ImplicitPointer: + return llvm::createStringError( + "DW_OP_deref_size of an implicit pointer is not supported"); + case Value::ValueType::Invalid: return llvm::createStringError("invalid value for DW_OP_deref_size"); @@ -1964,6 +2125,28 @@ llvm::Expected DWARFExpression::Evaluate( piece_byte_size, addr); } break; + case Value::ValueType::ImplicitPointer: { + // The piece is an implicit pointer: it has no actual bytes. + // Append placeholder bytes so that subsequent piece offsets stay + // correct, and record the byte range in the composite's + // implicit-pointer table. The table entry, never the placeholder + // bytes, is the piece's value; consumers must consult the table + // before interpreting any bytes of the composite. + const std::optional &pointee = + curr_piece_source_value.GetImplicitPointer(); + if (!pointee) + return llvm::createStringError( + "implicit pointer piece does not identify its pointee"); + if (curr_piece.ResizeData(piece_byte_size) != piece_byte_size) + return llvm::createStringError( + "failed to resize the piece memory buffer for " + "DW_OP_piece(%" PRIu64 ")", + piece_byte_size); + ::memset(curr_piece.GetBuffer().GetBytes(), 0, piece_byte_size); + pieces.AddImplicitPointerPiece(op_piece_offset, piece_byte_size, + *pointee); + } break; + case Value::ValueType::Scalar: { uint32_t bit_size = piece_byte_size * 8; uint32_t bit_offset = 0; @@ -2048,6 +2231,12 @@ llvm::Expected DWARFExpression::Evaluate( "unable to extract DW_OP_bit_piece(bit_size = %" PRIu64 ", bit_offset = %" PRIu64 ") from an address value.", piece_bit_size, piece_bit_offset); + + case Value::ValueType::ImplicitPointer: + return llvm::createStringError( + "unable to extract DW_OP_bit_piece(bit_size = %" PRIu64 + ", bit_offset = %" PRIu64 ") from an implicit pointer.", + piece_bit_size, piece_bit_offset); } } break; @@ -2076,10 +2265,44 @@ llvm::Expected DWARFExpression::Evaluate( break; } - case DW_OP_implicit_pointer: { + // OPCODE: DW_OP_implicit_pointer (DW_OP_GNU_implicit_pointer is the + // legacy vendor extension it was standardized from) + // OPERANDS: 2 + // A reference to a DIE in the .debug_info section, the size of an + // offset in the DWARF format of the compile unit (4 bytes in the + // 32-bit DWARF format, 8 bytes in the 64-bit DWARF format) + // SLEB128 byte offset into the value of that DIE + // DESCRIPTION: Specifies that the object is a pointer that cannot be + // represented as a real pointer, even though the value it would point to + // can be described: the referenced DIE describes the pointed-at value + // via its DW_AT_location (or DW_AT_const_value) attribute. The resulting + // value carries the DIE reference and byte offset so that the pointed-at + // value can be materialized on demand with + // DWARFExpression::DereferenceImplicitPointer(). The pointer itself has + // no numeric value; in particular it is distinct from a pointer whose + // value is zero. + case DW_OP_implicit_pointer: + case DW_OP_GNU_implicit_pointer: { dwarf4_location_description_kind = Implicit; - return llvm::createStringError("Could not evaluate %s.", - DW_OP_value_to_name(op)); + if (!dwarf_cu) + return llvm::createStringError( + "%s found without a compile unit being specified", + DW_OP_value_to_name(op)); + + // The DIE reference operand has the size of an offset in the DWARF + // format of the compile unit, except for the GNU extension in + // pre-DWARF 3 units, which had no notion of an offset sized + // reference and used an address sized one. + uint32_t ref_byte_size = dwarf_cu->GetDWARFOffsetByteSize(); + if (op == DW_OP_GNU_implicit_pointer && dwarf_cu->GetVersion() < 3) + ref_byte_size = opcodes.GetAddressByteSize(); + const uint64_t die_offset = opcodes.GetMaxU64(&offset, ref_byte_size); + const int64_t byte_offset = opcodes.GetSLEB128(&offset); + + Value result; + result.SetImplicitPointer(dwarf_cu, die_offset, byte_offset); + stack.push_back(result); + break; } // OPCODE: DW_OP_push_object_address @@ -2099,37 +2322,21 @@ llvm::Expected DWARFExpression::Evaluate( } break; - // OPCODE: DW_OP_call2 - // OPERANDS: - // uint16_t compile unit relative offset of a DIE - // DESCRIPTION: Performs subroutine calls during evaluation - // of a DWARF expression. The operand is the 2-byte unsigned offset of a - // debugging information entry in the current compilation unit. - // - // Operand interpretation is exactly like that for DW_FORM_ref2. - // - // This operation transfers control of DWARF expression evaluation to the - // DW_AT_location attribute of the referenced DIE. If there is no such - // attribute, then there is no effect. Execution of the DWARF expression of - // a DW_AT_location attribute may add to and/or remove from values on the - // stack. Execution returns to the point following the call when the end of - // the attribute is reached. Values on the stack at the time of the call - // may be used as parameters by the called expression and values left on - // the stack by the called expression may be used as return values by prior - // agreement between the calling and called expressions. - case DW_OP_call2: - return llvm::createStringError("unimplemented opcode DW_OP_call2"); - // OPCODE: DW_OP_call4 + // OPCODE: DW_OP_call2, DW_OP_call4, DW_OP_call_ref // OPERANDS: 1 - // uint32_t compile unit relative offset of a DIE - // DESCRIPTION: Performs a subroutine call during evaluation of a DWARF - // expression. For DW_OP_call4, the operand is a 4-byte unsigned offset of - // a debugging information entry in the current compilation unit. + // DW_OP_call2: uint16_t compile unit relative offset of a DIE + // DW_OP_call4: uint32_t compile unit relative offset of a DIE + // DW_OP_call_ref: offset of a DIE in the .debug_info section; the + // operand is 4 bytes in the 32-bit DWARF format and 8 bytes in + // the 64-bit DWARF format + // DESCRIPTION: Performs a DWARF procedure call during evaluation of a + // DWARF expression. // - // Operand interpretation DW_OP_call4 is exactly like that for - // DW_FORM_ref4. + // Operand interpretation of DW_OP_call2, DW_OP_call4 and DW_OP_call_ref + // is exactly like that for DW_FORM_ref2, DW_FORM_ref4 and + // DW_FORM_ref_addr, respectively. // - // This operation transfers control of DWARF expression evaluation to the + // These operations transfer control of DWARF expression evaluation to the // DW_AT_location attribute of the referenced DIE. If there is no such // attribute, then there is no effect. Execution of the DWARF expression of // a DW_AT_location attribute may add to and/or remove from values on the @@ -2138,8 +2345,61 @@ llvm::Expected DWARFExpression::Evaluate( // may be used as parameters by the called expression and values left on // the stack by the called expression may be used as return values by prior // agreement between the calling and called expressions. + case DW_OP_call2: case DW_OP_call4: - return llvm::createStringError("unimplemented opcode DW_OP_call4"); + case DW_OP_call_ref: { + if (!dwarf_cu) + return llvm::createStringError( + "%s found without a compile unit being specified", + DW_OP_value_to_name(op)); + + uint64_t die_offset; + bool unit_relative = true; + if (op == DW_OP_call2) + die_offset = opcodes.GetU16(&offset); + else if (op == DW_OP_call4) + die_offset = opcodes.GetU32(&offset); + else { + // The DW_OP_call_ref operand is an offset in the .debug_info + // section, whose size depends on the DWARF format of the compile + // unit. + die_offset = + opcodes.GetMaxU64(&offset, dwarf_cu->GetDWARFOffsetByteSize()); + unit_relative = false; + } + + // The current PC selects the applicable entry when the referenced + // DIE's DW_AT_location is a location list rather than a single + // expression. + auto callee_or_err = dwarf_cu->GetDIELocationExpression( + die_offset, unit_relative, + GetPCFunctionOffsetForLocationLists(exe_ctx, reg_ctx)); + if (!callee_or_err) + return callee_or_err.takeError(); + const DataExtractor &callee_opcodes = callee_or_err->first; + const DWARFExpression::Delegate *callee_cu = callee_or_err->second; + + // The referenced DIE has no DW_AT_location attribute; per the DWARF + // specification the call operation has no effect. + if (callee_opcodes.GetByteSize() == 0) + break; + + if (call_depth >= g_max_dwarf_call_depth) + return llvm::createStringError( + "DWARF expression call depth exceeds %u, likely due to a cycle " + "of DW_OP_call references", + g_max_dwarf_call_depth); + + // Evaluate the called expression against the current state, as if its + // opcodes appeared in place of the call operation. In particular it + // may consume and push values on the current stack. + if (llvm::Error err = EvaluateOpcodes( + exe_ctx, reg_ctx, module_sp, callee_opcodes, callee_cu, reg_kind, + object_address_ptr, stack, dwarf4_location_description_kind, + pieces, op_piece_offset, call_depth + 1, entry_value_ctx)) + return err; + break; + } // OPCODE: DW_OP_stack_value // OPERANDS: None @@ -2147,7 +2407,20 @@ llvm::Expected DWARFExpression::Evaluate( // rather is a constant value. The value from the top of the stack is the // value to be used. This is the actual object value and not the location. case DW_OP_stack_value: + if (stack.empty()) + return llvm::createStringError( + "expression stack needs at least 1 item for DW_OP_stack_value"); dwarf4_location_description_kind = Implicit; + // The value may be an as-yet unresolved file address (e.g. DW_OP_addr + // ... DW_OP_stack_value, describing a pointer to a statically + // allocated object). DW_OP_addr pushes an address "relocated by the + // consumer"; complete that deferred relocation now, before the + // conversion to a plain scalar erases the value's address provenance. + // Otherwise consumers would receive the unrelocated address (under a + // Darwin debug map, an address in the .o file's own address space). + // If the address cannot be resolved (no target, or the module is not + // loaded), the value is left as the file address, as before. + stack.back().ConvertToLoadAddress(module_sp.get(), target); stack.back().SetValueType(Value::ValueType::Scalar); break; @@ -2292,8 +2565,9 @@ llvm::Expected DWARFExpression::Evaluate( case DW_OP_GNU_entry_value: case DW_OP_entry_value: { - if (llvm::Error err = Evaluate_DW_OP_entry_value(stack, exe_ctx, reg_ctx, - opcodes, offset, log)) + if (llvm::Error err = + Evaluate_DW_OP_entry_value(stack, exe_ctx, reg_ctx, opcodes, + offset, log, entry_value_ctx)) return llvm::createStringError( "could not evaluate DW_OP_entry_value: %s", llvm::toString(std::move(err)).c_str()); @@ -2311,6 +2585,43 @@ llvm::Expected DWARFExpression::Evaluate( } } + return llvm::Error::success(); +} + +llvm::Expected DWARFExpression::Evaluate( + ExecutionContext *exe_ctx, RegisterContext *reg_ctx, + lldb::ModuleSP module_sp, const DataExtractor &opcodes, + const DWARFExpression::Delegate *dwarf_cu, + const lldb::RegisterKind reg_kind, const Value *initial_value_ptr, + const Value *object_address_ptr, + const EntryValueResolutionContext *entry_value_ctx) { + + if (opcodes.GetByteSize() == 0) + return llvm::createStringError( + "no location, value may have been optimized out"); + + Stack stack; + + if (initial_value_ptr) + stack.push_back(*initial_value_ptr); + + /// Insertion point for evaluating multi-piece expression. + uint64_t op_piece_offset = 0; + Value pieces; // Used for DW_OP_piece + + // The default kind is a memory location. This is updated by any + // operation that changes this, such as DW_OP_stack_value, and reset + // by composition operations like DW_OP_piece. + LocationDescriptionKind dwarf4_location_description_kind = Memory; + + if (llvm::Error err = EvaluateOpcodes( + exe_ctx, reg_ctx, module_sp, opcodes, dwarf_cu, reg_kind, + object_address_ptr, stack, dwarf4_location_description_kind, pieces, + op_piece_offset, /*call_depth=*/0, entry_value_ctx)) + return std::move(err); + + Log *log = GetLog(LLDBLog::Expressions); + if (stack.empty()) { // Nothing on the stack, check if we created a piece value from DW_OP_piece // or DW_OP_bit_piece opcodes @@ -2337,6 +2648,120 @@ llvm::Expected DWARFExpression::Evaluate( return stack.back(); } +llvm::Expected DWARFExpression::DereferenceImplicitPointer( + const Value &implicit_pointer, ExecutionContext *exe_ctx, + RegisterContext *reg_ctx, lldb::ModuleSP module_sp) { + if (implicit_pointer.GetValueType() != Value::ValueType::ImplicitPointer) + return llvm::createStringError( + "cannot dereference a value that is not an implicit pointer"); + const std::optional &info = + implicit_pointer.GetImplicitPointer(); + if (!info || !info->delegate) + return llvm::createStringError( + "implicit pointer does not identify a DWARF unit"); + + // The current PC selects the applicable entry when the pointee DIE's + // DW_AT_location is a location list rather than a single expression. + auto pointee_loc_or_err = info->delegate->GetDIELocationExpression( + info->die_offset, /*unit_relative=*/false, + GetPCFunctionOffsetForLocationLists(exe_ctx, reg_ctx)); + if (!pointee_loc_or_err) + return pointee_loc_or_err.takeError(); + const DataExtractor &pointee_opcodes = pointee_loc_or_err->first; + const Delegate *pointee_cu = pointee_loc_or_err->second; + + Value pointee; + if (pointee_opcodes.GetByteSize() != 0) { + llvm::Expected evaluated = + Evaluate(exe_ctx, reg_ctx, module_sp, pointee_opcodes, pointee_cu, + lldb::eRegisterKindDWARF, /*initial_value_ptr=*/nullptr, + /*object_address_ptr=*/nullptr); + if (!evaluated) + return evaluated.takeError(); + pointee = std::move(*evaluated); + } else { + // The DWARF specification allows the referenced DIE to describe the + // pointed-at value with a DW_AT_const_value attribute instead of a + // DW_AT_location. + llvm::Expected> const_value = + info->delegate->GetDIEConstValue(info->die_offset); + if (!const_value) + return const_value.takeError(); + if (!*const_value) + // The pointed-at value is unavailable; report that explicitly so that + // it can never be conflated with a value that happens to be zero. + return llvm::createStringError( + "implicit pointer target DIE at .debug_info offset 0x%" PRIx64 + " has no DW_AT_location applicable at the current PC and no " + "DW_AT_const_value; the pointed-at value is unavailable", + info->die_offset); + pointee = std::move(**const_value); + } + + const int64_t byte_offset = info->byte_offset; + switch (pointee.GetValueType()) { + case Value::ValueType::FileAddress: + case Value::ValueType::LoadAddress: + // The pointed-at value exists in the target's memory after all, so the + // result is an ordinary address there. + if (byte_offset != 0) + pointee.GetScalar() += byte_offset; + return pointee; + + case Value::ValueType::HostAddress: { + // The pointed-at value was materialized into a buffer that lives in + // LLDB's own memory, not the target's. Slice off the requested byte + // offset into a fresh value: Value's copy semantics only preserve + // scalars that point at the start of their own buffer, so we must not + // hand out a pointer into the middle of one. + if (byte_offset == 0) + return pointee; + const uint8_t *bytes = pointee.GetBuffer().GetBytes(); + const uint64_t byte_size = pointee.GetBuffer().GetByteSize(); + if (!bytes || byte_offset < 0 || + static_cast(byte_offset) >= byte_size) + return llvm::createStringError( + "implicit pointer byte offset %" PRId64 + " is outside the materialized value (%" PRIu64 " bytes)", + byte_offset, byte_size); + Value sliced(bytes + byte_offset, static_cast(byte_size - byte_offset)); + // Rebase the composite's implicit-pointer table onto the slice. Pieces + // entirely before the slice fall away with their bytes; a piece + // straddling the slice boundary would leave unmarked placeholder bytes + // in the slice, so refuse it. + const uint64_t slice_begin = static_cast(byte_offset); + for (const Value::ImplicitPointerPiece &piece : + pointee.GetImplicitPointerPieces()) { + if (piece.buffer_offset >= slice_begin) + sliced.AddImplicitPointerPiece(piece.buffer_offset - slice_begin, + piece.byte_size, piece.pointee); + else if (piece.buffer_offset + piece.byte_size > slice_begin) + return llvm::createStringError( + "implicit pointer byte offset %" PRId64 + " splits an implicit pointer piece of the pointed-at value", + byte_offset); + } + return sliced; + } + + case Value::ValueType::Scalar: + case Value::ValueType::ImplicitPointer: + // A scalar (e.g. a pointee location ending in DW_OP_stack_value, or an + // integer DW_AT_const_value) or a chained implicit pointer has no + // addressable bytes to offset into. + if (byte_offset != 0) + return llvm::createStringError( + "cannot apply implicit pointer byte offset %" PRId64 " to a %s", + byte_offset, Value::GetValueTypeAsCString(pointee.GetValueType())); + return pointee; + + case Value::ValueType::Invalid: + return llvm::createStringError( + "implicit pointer target evaluated to an invalid value"); + } + llvm_unreachable("all value types handled"); +} + bool DWARFExpression::MatchesOperand( StackFrame &frame, const Instruction::Operand &operand) const { using namespace OperandMatchers; diff --git a/lldb/source/Expression/DWARFExpressionList.cpp b/lldb/source/Expression/DWARFExpressionList.cpp index ef7333518f008..b99634849478b 100644 --- a/lldb/source/Expression/DWARFExpressionList.cpp +++ b/lldb/source/Expression/DWARFExpressionList.cpp @@ -11,6 +11,8 @@ #include "lldb/Symbol/Function.h" #include "lldb/Target/RegisterContext.h" #include "lldb/Target/StackFrame.h" +#include "lldb/Utility/LLDBLog.h" +#include "lldb/Utility/Log.h" #include "llvm/DebugInfo/DWARF/DWARFDebugLoc.h" #include "llvm/DebugInfo/DWARF/DWARFFormValue.h" @@ -144,6 +146,37 @@ bool DWARFExpressionList::LinkThreadLocalStorage( return true; } +bool DWARFExpressionList::LinkDWOPAddrOperands( + std::function const + &link_address_callback) { + bool all_linked = true; + for (size_t i = 0; i < m_exprs.GetSize(); ++i) { + DWARFExpression &expr = m_exprs.GetMutableEntryAtIndex(i)->data; + llvm::Expected file_addr = + expr.GetLocation_DW_OP_addr(m_dwarf_cu); + if (!file_addr) { + LLDB_LOG_ERROR(GetLog(LLDBLog::Expressions), file_addr.takeError(), + "Cannot link DW_OP_addr operand: {0}"); + all_linked = false; + continue; + } + if (*file_addr == LLDB_INVALID_ADDRESS) + // The expression has no DW_OP_addr operation. + continue; + const lldb::addr_t linked_file_addr = link_address_callback(*file_addr); + if (linked_file_addr == LLDB_INVALID_ADDRESS) { + // The object the address refers to did not make it into the linked + // executable. Leave the expression untouched; evaluating it will not + // produce a resolvable address. + all_linked = false; + continue; + } + if (!expr.Update_DW_OP_addr(m_dwarf_cu, linked_file_addr)) + all_linked = false; + } + return all_linked; +} + bool DWARFExpressionList::MatchesOperand( StackFrame &frame, const Instruction::Operand &operand) const { RegisterContextSP reg_ctx_sp = frame.GetRegisterContext(); @@ -232,7 +265,8 @@ void DWARFExpressionList::GetDescription(Stream *s, llvm::Expected DWARFExpressionList::Evaluate( ExecutionContext *exe_ctx, RegisterContext *reg_ctx, lldb::addr_t func_load_addr, const Value *initial_value_ptr, - const Value *object_address_ptr) const { + const Value *object_address_ptr, + const EntryValueResolutionContext *entry_value_ctx) const { ModuleSP module_sp = m_module_wp.lock(); DataExtractor data; RegisterKind reg_kind; @@ -267,5 +301,5 @@ llvm::Expected DWARFExpressionList::Evaluate( reg_kind = expr.GetRegisterKind(); return DWARFExpression::Evaluate(exe_ctx, reg_ctx, module_sp, data, m_dwarf_cu, reg_kind, initial_value_ptr, - object_address_ptr); + object_address_ptr, entry_value_ctx); } diff --git a/lldb/source/Plugins/Language/OxCaml/CMakeLists.txt b/lldb/source/Plugins/Language/OxCaml/CMakeLists.txt index 1aa3cbe45afec..d76eff3ff0c95 100644 --- a/lldb/source/Plugins/Language/OxCaml/CMakeLists.txt +++ b/lldb/source/Plugins/Language/OxCaml/CMakeLists.txt @@ -1,13 +1,29 @@ +lldb_tablegen(LanguageOxCamlProperties.inc -gen-lldb-property-defs + SOURCE LanguageOxCamlProperties.td + TARGET LLDBPluginLanguageOxCamlPropertiesGen) + +lldb_tablegen(LanguageOxCamlPropertiesEnum.inc -gen-lldb-property-enum-defs + SOURCE LanguageOxCamlProperties.td + TARGET LLDBPluginLanguageOxCamlPropertiesEnumGen) + add_lldb_library(lldbPluginOxCamlLanguage PLUGIN LLDBFormatHelpersForOxCaml.cpp OxCamlLanguage.cpp + OxCamlExternalPrinter.cpp OxCamlFormatters.cpp OxCamlFormatHelpers.cpp + OxCamlMarshal.cpp OxCamlValueFormatters.cpp LogChannelOxCaml.cpp LINK_LIBS lldbCore + lldbDataFormatters + lldbExpression lldbTarget lldbUtility ) + +add_dependencies(lldbPluginOxCamlLanguage + LLDBPluginLanguageOxCamlPropertiesGen + LLDBPluginLanguageOxCamlPropertiesEnumGen) diff --git a/lldb/source/Plugins/Language/OxCaml/LanguageOxCamlProperties.td b/lldb/source/Plugins/Language/OxCaml/LanguageOxCamlProperties.td new file mode 100644 index 0000000000000..ce2a2a5617061 --- /dev/null +++ b/lldb/source/Plugins/Language/OxCaml/LanguageOxCamlProperties.td @@ -0,0 +1,15 @@ +include "../../../../include/lldb/Core/PropertiesBase.td" + +let Definition = "language_oxcaml" in { + def ExternalSummaryExecutable: + Property<"external-summary-executable", "FileSpec">, + Global, + DefaultStringValue<"">, + Desc<"Path to an executable used to pretty-print OCaml values. The " + "executable is run with the OCaml type name (e.g. Env.t) as its only " + "argument and receives the value, marshalled in OCaml's Marshal wire " + "format, on its standard input. If it recognizes the type it must print " + "a rendering of the value on its standard output and exit with code 0; " + "any other exit code makes LLDB fall back to the built-in OCaml " + "formatter. When empty, the built-in formatter is always used.">; +} diff --git a/lldb/source/Plugins/Language/OxCaml/LogChannelOxCaml.h b/lldb/source/Plugins/Language/OxCaml/LogChannelOxCaml.h index 4587a27adeea6..0529a708cd7f5 100644 --- a/lldb/source/Plugins/Language/OxCaml/LogChannelOxCaml.h +++ b/lldb/source/Plugins/Language/OxCaml/LogChannelOxCaml.h @@ -29,12 +29,17 @@ LLVM_ENABLE_BITMASK_ENUMS_IN_NAMESPACE(); #define OXCAML_EXPLAIN_ERROR_MARKER(DISPLAY_TOKEN, FORMAT_STRING, ...) \ do { \ + /* The format arguments must be evaluated exactly once, whether or not \ + the log channel is enabled: call sites pass expressions with side \ + effects, such as llvm::toString(expected.takeError()), which must run \ + to consume the error (an unconsumed llvm::Expected error aborts). */ \ + std::string oxcaml_explanation__ = \ + llvm::formatv(FORMAT_STRING, __VA_ARGS__).str(); \ ::lldb_private::Log *oxcaml_log__ = \ ::lldb_private::GetLog(::lldb_private::OxCamlLog::UserVisibleErrors); \ if (oxcaml_log__) \ - LLDB_LOG(oxcaml_log__, "{0}; displaying {1} ({2})", \ - llvm::formatv(FORMAT_STRING, __VA_ARGS__), (DISPLAY_TOKEN), \ - __func__); \ + LLDB_LOG(oxcaml_log__, "{0}; displaying {1} ({2})", oxcaml_explanation__, \ + (DISPLAY_TOKEN), __func__); \ } while (false) #define OXCAML_EMIT_MARKER(STREAM, MARKER_EXPR, ERROR_FMT, ...) \ diff --git a/lldb/source/Plugins/Language/OxCaml/OxCamlExternalPrinter.cpp b/lldb/source/Plugins/Language/OxCaml/OxCamlExternalPrinter.cpp new file mode 100644 index 0000000000000..ae6edd6e9a3e4 --- /dev/null +++ b/lldb/source/Plugins/Language/OxCaml/OxCamlExternalPrinter.cpp @@ -0,0 +1,137 @@ +//===-- OxCamlExternalPrinter.cpp -----------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "OxCamlExternalPrinter.h" +#include "LogChannelOxCaml.h" +#include "OxCamlLanguage.h" +#include "OxCamlMarshal.h" +#include "Plugins/TypeSystem/OxCaml/TypeSystemOxCaml.h" +#include "lldb/Core/Value.h" +#include "lldb/DataFormatters/TypeSummary.h" +#include "lldb/Symbol/CompilerType.h" +#include "lldb/Target/Process.h" +#include "lldb/Utility/DataExtractor.h" +#include "lldb/Utility/FileSpec.h" +#include "lldb/Utility/Log.h" +#include "lldb/Utility/Reference.h" +#include "lldb/Utility/Status.h" +#include "lldb/Utility/Stream.h" +#include "lldb/ValueObject/ValueObject.h" +#include "llvm/ADT/StringRef.h" + +using namespace lldb; +using namespace lldb_private; +using namespace lldb_private::formatters::oxcaml; + +static OxCamlType *ResolveTypedefs(OxCamlType *type) { + while (type && type->GetKind() == OxCamlType::Typedef) + type = static_cast(type)->GetUnderlyingType(); + return type; +} + +llvm::Expected +lldb_private::formatters::oxcaml::GetOCamlExternalFormatterInput( + ValueObject &valobj, std::vector &data) { + CompilerType compiler_type = valobj.GetCompilerType(); + auto *type_ref = + static_cast *>(compiler_type.GetOpaqueQualType()); + if (!type_ref || !type_ref->get()) + return llvm::createStringError("value has no OxCaml type information"); + + // Only values using the tagged ocaml_value representation can be + // marshalled; unboxed primitives (float#, int64#, ...) are raw bits whose + // low bit means nothing, and unresolved types tell us nothing at all. + OxCamlType *type = ResolveTypedefs(type_ref->get()); + if (!type) + return llvm::createStringError("value has an unresolved typedef chain"); + switch (type->GetKind()) { + case OxCamlType::UnboxedBase: + case OxCamlType::Placeholder: + case OxCamlType::Unknown: + return llvm::createStringError( + "type '%s' is not represented as a tagged OCaml value", + type->GetDisplayName().c_str()); + default: + break; + } + + // Values described by DWARF implicit pointers (in whole or in part) have + // no bytes of their own in the debuggee, so the marshaller cannot walk + // them yet; a MarshalMemoryReader materializing them from the DWARF + // description would lift this restriction. + if (valobj.GetValue().GetValueType() == Value::ValueType::ImplicitPointer || + !valobj.GetValue().GetImplicitPointerPieces().empty()) + return llvm::createStringError( + "value is described by an implicit pointer"); + + lldb::ProcessSP process_sp = valobj.GetProcessSP(); + if (!process_sp) + return llvm::createStringError("value has no process"); + + DataExtractor extractor; + Status status; + valobj.GetData(extractor, status); + if (status.Fail()) + return llvm::createStringError("cannot get value data: %s", + status.AsCString()); + if (extractor.GetByteSize() < sizeof(uint64_t)) + return llvm::createStringError( + "value has %zu byte(s) of data, expected at least 8", + static_cast(extractor.GetByteSize())); + + lldb::offset_t offset = 0; + const uint64_t root = extractor.GetU64(&offset); + + ProcessMemoryReader reader(process_sp); + llvm::Expected> marshalled = + MarshalOCamlValue(root, reader); + if (!marshalled) + return marshalled.takeError(); + data = std::move(*marshalled); + + // "Env.t @ value" -> "Env.t": the layout annotation is an artifact of the + // DWARF encoding, not part of the OCaml type name the printer knows. + llvm::StringRef name = valobj.GetDisplayTypeName().GetStringRef(); + size_t layout_pos = name.rfind(" @ "); + if (layout_pos != llvm::StringRef::npos) + name = name.take_front(layout_pos); + return name.str(); +} + +bool lldb_private::formatters::oxcaml::TryExternalPrettyPrinter( + ValueObject &valobj, Stream &stream) { + FileSpec executable = OxCamlLanguage::GetExternalSummaryExecutable(); + if (!executable) + return false; + + Log *log = GetLog(OxCamlLog::Formatting); + + std::vector data; + llvm::Expected type_name = + GetOCamlExternalFormatterInput(valobj, data); + if (!type_name) { + LLDB_LOG_ERROR(log, type_name.takeError(), + "external printer: cannot marshal value: {0}"); + return false; + } + + LLDB_LOG(log, "external printer: running '{0}' for type '{1}' with {2} " + "marshalled byte(s)", + executable.GetPath(), *type_name, data.size()); + + llvm::Expected output = + ExternalSummaryFormat::RunSummaryExecutable(executable.GetPath(), + *type_name, data); + if (!output) { + LLDB_LOG_ERROR(log, output.takeError(), "external printer: {0}"); + return false; + } + + stream.PutCString(*output); + return true; +} diff --git a/lldb/source/Plugins/Language/OxCaml/OxCamlExternalPrinter.h b/lldb/source/Plugins/Language/OxCaml/OxCamlExternalPrinter.h new file mode 100644 index 0000000000000..000921e4f7b04 --- /dev/null +++ b/lldb/source/Plugins/Language/OxCaml/OxCamlExternalPrinter.h @@ -0,0 +1,58 @@ +//===-- OxCamlExternalPrinter.h ---------------------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// Support for pretty-printing OCaml values with a user-provided external +// program. The value is marshalled out of the debuggee (see OxCamlMarshal.h) +// and sent to the program on its standard input, together with the OCaml +// type name as the program's only argument; the program demarshals the value +// (Marshal.from_channel), prints its rendering on standard output and exits +// with code 0. A nonzero exit code means "type not recognized" and makes +// the caller fall back to the built-in formatter. +// +//===----------------------------------------------------------------------===// + +#ifndef LLDB_SOURCE_PLUGINS_LANGUAGE_OXCAML_OXCAMLEXTERNALPRINTER_H +#define LLDB_SOURCE_PLUGINS_LANGUAGE_OXCAML_OXCAMLEXTERNALPRINTER_H + +#include "lldb/lldb-forward.h" +#include "llvm/Support/Error.h" +#include +#include +#include + +namespace lldb_private { + +class Stream; +class ValueObject; + +namespace formatters { +namespace oxcaml { + +/// Serialize \p valobj for an external summary program: marshal the OCaml +/// value into \p data and return the OCaml type name to pass to the program +/// (the display type name with its layout annotation stripped, e.g. +/// "Env.t @ value" becomes "Env.t"). Fails for values that are not +/// represented as tagged OCaml words (unboxed primitives), for values +/// without a live process, and for object graphs the marshaller cannot +/// represent (closures, custom blocks, unreadable memory, ...). +llvm::Expected +GetOCamlExternalFormatterInput(ValueObject &valobj, std::vector &data); + +/// If the user has configured an external pretty-printer executable +/// (settings set plugin.oxcaml.display.external-summary-executable ), +/// try to use it for \p valobj, writing its output to \p stream. Returns +/// true on success; any failure returns false (with the reason logged to +/// the "oxcaml formatting" log channel) so the caller can fall back to the +/// built-in formatter. +bool TryExternalPrettyPrinter(ValueObject &valobj, Stream &stream); + +} // namespace oxcaml +} // namespace formatters +} // namespace lldb_private + +#endif // LLDB_SOURCE_PLUGINS_LANGUAGE_OXCAML_OXCAMLEXTERNALPRINTER_H diff --git a/lldb/source/Plugins/Language/OxCaml/OxCamlFormatters.cpp b/lldb/source/Plugins/Language/OxCaml/OxCamlFormatters.cpp index c01caaa4e01fc..edd5afb1f5350 100644 --- a/lldb/source/Plugins/Language/OxCaml/OxCamlFormatters.cpp +++ b/lldb/source/Plugins/Language/OxCaml/OxCamlFormatters.cpp @@ -49,17 +49,23 @@ #include "OxCamlFormatters.h" #include "LogChannelOxCaml.h" #include "OxCamlAssert.h" +#include "OxCamlExternalPrinter.h" #include "OxCamlFormatHelpers.h" #include "OxCamlHelpers.h" #include "OxCamlValueFormatters.h" #include "Plugins/TypeSystem/OxCaml/TypeSystemOxCaml.h" +#include "lldb/Core/Value.h" +#include "lldb/Expression/DWARFExpression.h" #include "lldb/Symbol/CompilerType.h" #include "lldb/Target/Process.h" +#include "lldb/Target/RegisterContext.h" +#include "lldb/Target/StackFrame.h" #include "lldb/Utility/DataExtractor.h" #include "lldb/Utility/Log.h" #include "lldb/Utility/Reference.h" #include "lldb/Utility/Status.h" #include "lldb/ValueObject/ValueObject.h" +#include "llvm/ADT/SmallVector.h" #include "llvm/ADT/StringRef.h" #include "llvm/Support/FormatVariadic.h" #include @@ -86,50 +92,60 @@ static uint64_t MakeLowBitMask(uint64_t bit_size) { return (1ULL << bit_size) - 1ULL; } - +// Most formatters receive, alongside [data], the implicit-pointer piece +// table of the composite the bytes came from (empty for values read from +// the target's memory). Ranges listed there contain placeholder bytes, not +// data: their value is the table entry. See FormatImplicitPointer. static bool FormatValue(Stream &stream, OxCamlType *type, DataExtractor &data, - lldb::ProcessSP process_sp, - const ExecutionContextRef &exe_ctx_ref); + const OxCamlFormatContext &context, + std::optional object_address, + llvm::ArrayRef + implicit_pieces = {}); +static bool FormatImplicitPointer(Stream &stream, OxCamlType *type, + const Value &pointer, + const OxCamlFormatContext &context, + unsigned depth = 0); static bool FormatUnboxedBase(Stream &stream, OxCamlUnboxedBaseType *unboxed_type, - DataExtractor &data, lldb::ProcessSP process_sp); + DataExtractor &data); static bool FormatFallback(Stream &stream, OxCamlType *type, - DataExtractor &data, lldb::ProcessSP process_sp); + DataExtractor &data); static bool FormatEnum(Stream &stream, OxCamlEnumType *enum_type, - DataExtractor &data, lldb::ProcessSP process_sp); + DataExtractor &data); static bool FormatPointer(Stream &stream, OxCamlPointerType *ptr_type, - DataExtractor &data, lldb::ProcessSP process_sp, - const ExecutionContextRef &exe_ctx_ref); + DataExtractor &data, + const OxCamlFormatContext &context); static bool FormatTypedef(Stream &stream, OxCamlTypedefType *typedef_type, - DataExtractor &data, lldb::ProcessSP process_sp, - const ExecutionContextRef &exe_ctx_ref); + DataExtractor &data, + const OxCamlFormatContext &context, + std::optional object_address, + llvm::ArrayRef + implicit_pieces = {}); static bool FormatStructure(Stream &stream, OxCamlStructureType *struct_type, - DataExtractor &data, lldb::ProcessSP process_sp, - const ExecutionContextRef &exe_ctx_ref); + DataExtractor &data, + const OxCamlFormatContext &context, + std::optional object_address, + llvm::ArrayRef + implicit_pieces = {}); static bool FormatPlaceholder(Stream &stream, OxCamlPlaceholderType *placeholder_type, - DataExtractor &data, lldb::ProcessSP process_sp); + DataExtractor &data); static bool FormatUnknown(Stream &stream, OxCamlUnknownType *unknown_type, - DataExtractor &data, lldb::ProcessSP process_sp); + DataExtractor &data); static bool FormatArray(Stream &stream, OxCamlArrayType *array_type, - DataExtractor &data, lldb::ProcessSP process_sp, - const ExecutionContextRef &exe_ctx_ref); - -// Forward declarations for variant functions called from -// FormatPointer/FormatStructure -static uint64_t -CalculateMinimumSizeForDiscriminators(OxCamlStructureType *struct_type); -static uint64_t EstimatePointerAllocationSize(OxCamlType *type, - DataExtractor &data); -static uint64_t ComputeActualPointerSize(OxCamlType *pointed_to, - lldb::addr_t adjusted_address, - const DataExtractor &data, - lldb::ProcessSP process_sp); + DataExtractor &data, const OxCamlFormatContext &context, + std::optional object_address); +static bool TryFormatRuntimeFloatArray(Stream &stream, lldb::addr_t array_ptr, + lldb::ProcessSP process_sp); + static bool FormatVariantPart(Stream &stream, const OxCamlVariantPart &variant_part, - DataExtractor &data, lldb::ProcessSP process_sp, - const ExecutionContextRef &exe_ctx_ref, - bool is_ocaml_variant); + DataExtractor &data, + const OxCamlFormatContext &context, + std::optional object_address, + bool is_ocaml_variant, + llvm::ArrayRef + implicit_pieces = {}); // ============================================================================ // Entry Point: OxCamlValue_SummaryProvider @@ -144,13 +160,6 @@ static bool FormatVariantPart(Stream &stream, bool lldb_private::formatters::oxcaml::OxCamlValue_SummaryProvider( ValueObject &valobj, Stream &stream, const TypeSummaryOptions &options) { - DataExtractor data; - Status error; - valobj.GetData(data, error); - - ENSURE(error.Success(), stream, "", - "ValueObject data extraction failed: {0}", error.AsCString()); - CompilerType compiler_type = valobj.GetCompilerType(); void *opaque_type = compiler_type.GetOpaqueQualType(); @@ -173,9 +182,40 @@ bool lldb_private::formatters::oxcaml::OxCamlValue_SummaryProvider( "(ptr={0:P})", process_sp.get()); - const ExecutionContextRef &exe_ctx_ref = valobj.GetExecutionContextRef(); + // A user-configured external pretty-printer takes precedence; on any + // failure (unmarshallable value, unrecognized type, printer error) the + // built-in formatting below is used instead. + if (TryExternalPrettyPrinter(valobj, stream)) + return true; + + // The variable may have been optimized away into an implicit pointer + // (DW_OP_implicit_pointer): it then has no bytes of its own, and the + // value it points at must be materialized from the DWARF description of + // the pointee instead. + if (valobj.GetValue().GetValueType() == Value::ValueType::ImplicitPointer) { + OxCamlFormatContext context{process_sp, valobj.GetExecutionContextRef(), + process_sp->GetByteOrder(), + process_sp->GetAddressByteSize(), + valobj.GetModule()}; + return FormatImplicitPointer(stream, type, valobj.GetValue(), context); + } - return FormatValue(stream, type, data, process_sp, exe_ctx_ref); + DataExtractor data; + Status error; + valobj.GetData(data, error); + + ENSURE(error.Success(), stream, "", + "ValueObject data extraction failed: {0}", error.AsCString()); + + OxCamlFormatContext context{process_sp, valobj.GetExecutionContextRef(), + data.GetByteOrder(), data.GetAddressByteSize(), + valobj.GetModule()}; + + // No object address at entry: [data] holds the OCaml value itself + // (typically a register-resident tagged pointer). FormatPointer sets the + // object address for any pointed-to objects it dereferences. + return FormatValue(stream, type, data, context, + /*object_address=*/std::nullopt); } // ============================================================================ @@ -183,7 +223,7 @@ bool lldb_private::formatters::oxcaml::OxCamlValue_SummaryProvider( // ============================================================================ static bool FormatFallback(Stream &stream, OxCamlType *type, - DataExtractor &data, lldb::ProcessSP process_sp) { + DataExtractor &data) { size_t byte_size = data.GetByteSize(); if (byte_size == 0) { if (type) { @@ -217,7 +257,7 @@ static bool FormatFallback(Stream &stream, OxCamlType *type, static bool FormatPlaceholder(Stream &stream, OxCamlPlaceholderType *placeholder_type, - DataExtractor &data, lldb::ProcessSP process_sp) { + DataExtractor &data) { OXCAML_EMIT_MARKER( stream, "", "Cannot format unresolved placeholder type '{0}' (DIE 0x{1:x16})", @@ -226,7 +266,7 @@ static bool FormatPlaceholder(Stream &stream, } static bool FormatUnknown(Stream &stream, OxCamlUnknownType *unknown_type, - DataExtractor &data, lldb::ProcessSP process_sp) { + DataExtractor &data) { OXCAML_EMIT_MARKER( stream, "", "Unsupported DWARF tag 0x{0:x} for type '{1}' (DIE 0x{2:x16})", @@ -237,7 +277,7 @@ static bool FormatUnknown(Stream &stream, OxCamlUnknownType *unknown_type, static bool FormatUnboxedBase(Stream &stream, OxCamlUnboxedBaseType *unboxed_type, - DataExtractor &data, lldb::ProcessSP process_sp) { + DataExtractor &data) { uint64_t byte_size = unboxed_type->GetByteSize(); ENSURE(byte_size != 0, stream, "<0-byte base type>", "Unboxed base type '{0}' (DIE 0x{1:x}) reports zero byte size", @@ -299,7 +339,7 @@ static bool FormatUnboxedBase(Stream &stream, } static bool FormatEnum(Stream &stream, OxCamlEnumType *enum_type, - DataExtractor &data, lldb::ProcessSP process_sp) { + DataExtractor &data) { uint64_t byte_size = enum_type->GetByteSize(); OX_ASSERT(byte_size > 0 && byte_size <= helpers::constants::WORD_SIZE, "OCaml enum type '{0}' (DIE 0x{1:x}) has size {2} bytes, must be " @@ -325,9 +365,80 @@ static bool FormatEnum(Stream &stream, OxCamlEnumType *enum_type, return true; } +// Resolve the byte size of [type] at format time. For structures, evaluates +// the dynamic DW_AT_byte_size/DW_AT_bit_size against [object_address] (which +// must be the type's own object address). For arrays, compiler-emitted OxCaml +// DWARF wraps the DW_TAG_array_type in a reference type and equips the array +// with DW_AT_count and DW_AT_byte_stride. The count expression reads the OCaml +// block header wosize; generic runtime array discovery belongs to the +// ocaml_value formatter, not this typed-array path. For everything else, +// returns the static byte size. +static llvm::Expected +ResolveTypeByteSize(OxCamlType *type, + std::optional object_address, + const OxCamlFormatContext &context) { + OxCamlType *underlying = type; + while (underlying && underlying->GetKind() == OxCamlType::Typedef) + underlying = + static_cast(underlying)->GetUnderlyingType(); + if (!underlying) + return type->GetByteSize(); + + if (underlying->GetKind() == OxCamlType::Structure) { + auto *struct_type = static_cast(underlying); + auto result = + struct_type->GetDynamicSize().Evaluate(context, object_address); + if (!result) + return result.takeError(); + OX_ASSERT( + result->GetKind() == OxCamlLayoutValueKind::ScalarBytes || + result->GetKind() == OxCamlLayoutValueKind::ScalarBits, + "Dynamic size for structure type '{0}' (DIE 0x{1:x}) evaluated to " + "layout kind {2}; expected ScalarBytes or ScalarBits", + struct_type->GetDisplayName(), struct_type->GetDieId(), + static_cast(result->GetKind())); + if (result->GetKind() == OxCamlLayoutValueKind::ScalarBits) + return (result->GetScalar() + 7) / 8; + return result->GetScalar(); + } + + if (underlying->GetKind() == OxCamlType::Array) { + auto *array_type = static_cast(underlying); + auto stride_result = + array_type->GetStride().Evaluate(context, object_address); + if (!stride_result) + return stride_result.takeError(); + OX_ASSERT(stride_result->GetKind() == OxCamlLayoutValueKind::ScalarBytes, + "DW_AT_byte_stride for array '{0}' (DIE 0x{1:x}) evaluated to " + "layout kind {2}; expected ScalarBytes", + array_type->GetDisplayName(), array_type->GetDieId(), + static_cast(stride_result->GetKind())); + uint64_t stride = stride_result->GetScalar(); + + const auto &count = array_type->GetCount(); + if (!count.has_value()) + return llvm::createStringError( + "Array type '%s' (DIE 0x%" PRIx64 + ") has no DW_AT_count; compiler-emitted OxCaml arrays must include " + "a subrange count expression", + array_type->GetDisplayName().c_str(), array_type->GetDieId()); + auto count_result = count->Evaluate(context, object_address); + if (!count_result) + return count_result.takeError(); + OX_ASSERT(count_result->GetKind() == OxCamlLayoutValueKind::ScalarElements, + "DW_AT_count for array '{0}' (DIE 0x{1:x}) evaluated to layout " + "kind {2}; expected ScalarElements", + array_type->GetDisplayName(), array_type->GetDieId(), + static_cast(count_result->GetKind())); + return count_result->GetScalar() * stride; + } + + return type->GetByteSize(); +} + static bool FormatPointer(Stream &stream, OxCamlPointerType *ptr_type, - DataExtractor &data, lldb::ProcessSP process_sp, - const ExecutionContextRef &exe_ctx_ref) { + DataExtractor &data, + const OxCamlFormatContext &context) { uint64_t ptr_size = ptr_type->GetByteSize(); ENSURE(ptr_size == helpers::constants::WORD_SIZE, stream, "", "OCaml pointer type '{0}' (DIE 0x{1:x}) has size {2} bytes, " @@ -354,62 +465,173 @@ static bool FormatPointer(Stream &stream, OxCamlPointerType *ptr_type, "Pointer 0x{0:x} to '{1}' (DIE 0x{2:x}) lacks a resolved target type", ptr_value, ptr_type->GetDisplayName(), ptr_type->GetDieId()); - // Special case: Arrays are variable-sized and need the raw pointer value - // Don't dereference - just pass the data (containing pointer) to FormatArray - if (pointed_to->GetKind() == OxCamlType::Array) { - return FormatArray(stream, static_cast(pointed_to), data, - process_sp, exe_ctx_ref); - } - - uint64_t size = pointed_to->GetByteSize(); - ENSURE(size != 0, stream, "", - "Resolved target type '{0}' (DIE 0x{1:x}) reports byte size 0; " - "cannot dereference pointer 0x{2:x}", - pointed_to->GetDisplayName(), pointed_to->GetDieId(), ptr_value); - - // potentially offset the pointer for OCaml blocks - int64_t base_offset = pointed_to->GetPointerAdjustmentOffset(); - uint64_t adjusted_address = ptr_value + base_offset; - - uint64_t actual_size = - ComputeActualPointerSize(pointed_to, adjusted_address, data, process_sp); - - if (base_offset != 0) { - Log *log = GetLog(OxCamlLog::Formatting); - LLDB_LOG(log, - "FormatPointer: Applying base offset {0} to pointer 0x{1:x}, " - "adjusted address: 0x{2:x}, reading {3} bytes", - base_offset, ptr_value, adjusted_address, actual_size); + lldb::addr_t object_address = ptr_value; + + llvm::Expected actual_size_expected = + ResolveTypeByteSize(pointed_to, object_address, context); + if (!actual_size_expected) { + OXCAML_EMIT_MARKER( + stream, "", + "Failed to evaluate dynamic byte size for pointer 0x{0:x} to " + "'{1}' (DIE 0x{2:x}): {3}", + ptr_value, pointed_to->GetDisplayName(), pointed_to->GetDieId(), + llvm::toString(actual_size_expected.takeError())); + return true; } - + uint64_t actual_size = *actual_size_expected; + OxCamlType *underlying_pointed_to = pointed_to; + while (underlying_pointed_to && + underlying_pointed_to->GetKind() == OxCamlType::Typedef) + underlying_pointed_to = + static_cast(underlying_pointed_to) + ->GetUnderlyingType(); + bool is_array_target = underlying_pointed_to && + underlying_pointed_to->GetKind() == OxCamlType::Array; + + // Empty OCaml arrays are real heap blocks with zero-byte payloads. Keep the + // zero-size guard for other pointer targets, but let arrays flow through to + // FormatArray with an empty DataExtractor and the object address. ENSURE( - actual_size != 0, stream, "", + actual_size != 0 || is_array_target, stream, "", "Resolved target type '{0}' (DIE 0x{1:x}) computed dereference size 0; " "cannot dereference pointer 0x{2:x}", pointed_to->GetDisplayName(), pointed_to->GetDieId(), ptr_value); std::vector buffer(actual_size); - Status error; - size_t bytes_read = process_sp->ReadMemory(adjusted_address, buffer.data(), - actual_size, error); - - ENSURE(bytes_read == actual_size && error.Success(), stream, "", - "Failed to read {0} bytes from address 0x{1:x} for pointer 0x{2:x} " - "to '{3}' (DIE 0x{4:x})", - actual_size, adjusted_address, ptr_value, pointed_to->GetDisplayName(), - pointed_to->GetDieId()); - - DataExtractor pointed_data(buffer.data(), actual_size, data.GetByteOrder(), - data.GetAddressByteSize()); + DataExtractor pointed_data; + pointed_data.SetByteOrder(data.GetByteOrder()); + pointed_data.SetAddressByteSize(data.GetAddressByteSize()); + if (actual_size > 0) { + Status error; + size_t bytes_read = context.process_sp->ReadMemory( + object_address, buffer.data(), actual_size, error); + + ENSURE(bytes_read == actual_size && error.Success(), stream, "", + "Failed to read {0} bytes from address 0x{1:x} for pointer 0x{2:x} " + "to '{3}' (DIE 0x{4:x})", + actual_size, object_address, ptr_value, pointed_to->GetDisplayName(), + pointed_to->GetDieId()); + + pointed_data.SetData(buffer.data(), actual_size, data.GetByteOrder()); + } - return FormatValue(stream, pointed_to, pointed_data, process_sp, exe_ctx_ref); + return FormatValue(stream, pointed_to, pointed_data, context, object_address); } static bool FormatTypedef(Stream &stream, OxCamlTypedefType *typedef_type, - DataExtractor &data, lldb::ProcessSP process_sp, - const ExecutionContextRef &exe_ctx_ref) { - return FormatValue(stream, typedef_type->GetUnderlyingType(), data, - process_sp, exe_ctx_ref); + DataExtractor &data, + const OxCamlFormatContext &context, + std::optional object_address, + llvm::ArrayRef + implicit_pieces) { + return FormatValue(stream, typedef_type->GetUnderlyingType(), data, context, + object_address, implicit_pieces); +} + +// Format a value whose location is an implicit pointer +// (DW_OP_implicit_pointer): the pointer itself was optimized away and has no +// numeric value, but the object it would point at is described by another +// DIE. Materialize that object and format it as [type]'s pointee. Failures +// surface as markers; an unavailable pointee is never rendered as a null or +// zero pointer. +static bool FormatImplicitPointer(Stream &stream, OxCamlType *type, + const Value &pointer, + const OxCamlFormatContext &context, + unsigned depth) { + // Guard against reference cycles in (malformed) DWARF; each level of the + // chain materializes one pointee. + ENSURE(depth < 32, stream, "", + "Implicit pointer chain exceeds {0} levels, likely a cycle", 32); + + // Resolve typedef chains to find the pointer type whose pointee we will + // format. + OxCamlType *resolved = type; + while (resolved && resolved->GetKind() == OxCamlType::Typedef) + resolved = static_cast(resolved)->GetUnderlyingType(); + ENSURE(resolved, stream, "", + "Implicit pointer value has unresolved type '{0}'", + type ? type->GetDisplayName() : ""); + ENSURE(resolved->GetKind() == OxCamlType::Pointer, stream, + "", + "Implicit pointer value has non-pointer type '{0}'", + resolved->GetDisplayName()); + auto *ptr_type = static_cast(resolved); + OxCamlType *pointed_to = ptr_type->GetPointedToType(); + ENSURE(pointed_to, stream, "", + "Implicit pointer type '{0}' lacks a resolved target type", + ptr_type->GetDisplayName()); + + ExecutionContext exe_ctx(context.exe_ctx_ref); + RegisterContext *reg_ctx = nullptr; + if (StackFrame *frame = exe_ctx.GetFramePtr()) + reg_ctx = frame->GetRegisterContext().get(); + + llvm::Expected pointee = DWARFExpression::DereferenceImplicitPointer( + pointer, &exe_ctx, reg_ctx, context.module_sp); + if (!pointee) { + OXCAML_EMIT_MARKER(stream, "", + "Failed to materialize the value of an optimized-out " + "pointer of type '{0}': {1}", + ptr_type->GetDisplayName(), + llvm::toString(pointee.takeError())); + return true; + } + + switch (pointee->GetValueType()) { + case Value::ValueType::LoadAddress: { + // The pointed-at object exists in the target's memory after all: + // re-enter the ordinary pointer-chasing path with its load address. + const uint64_t addr = pointee->GetScalar().ULongLong(LLDB_INVALID_ADDRESS); + uint8_t addr_bytes[sizeof(uint64_t)]; + for (unsigned i = 0; i < sizeof(addr_bytes); ++i) + addr_bytes[i] = (context.byte_order == lldb::eByteOrderBig) + ? uint8_t(addr >> (8 * (sizeof(addr_bytes) - 1 - i))) + : uint8_t(addr >> (8 * i)); + DataExtractor addr_data(addr_bytes, sizeof(addr_bytes), context.byte_order, + context.address_byte_size); + return FormatPointer(stream, ptr_type, addr_data, context); + } + + case Value::ValueType::HostAddress: { + // The pointed-at object exists only in the debugger: format the block + // from the materialized buffer, together with its implicit-pointer + // table (fields of the block may themselves be optimized-out + // pointers). Member locations that are constant object-relative + // offsets resolve against the buffer; there is no target-side object + // address. + DataExtractor pointee_data(pointee->GetBuffer().GetBytes(), + pointee->GetBuffer().GetByteSize(), + context.byte_order, context.address_byte_size); + return FormatValue(stream, pointed_to, pointee_data, context, + /*object_address=*/std::nullopt, + pointee->GetImplicitPointerPieces()); + } + + case Value::ValueType::Scalar: { + // The pointed-at object was materialized as a single word (e.g. its + // location is a register or a DW_AT_const_value integer). + uint64_t word = pointee->GetScalar().ULongLong(); + DataExtractor word_data(&word, sizeof(word), context.byte_order, + context.address_byte_size); + return FormatValue(stream, pointed_to, word_data, context, + /*object_address=*/std::nullopt); + } + + case Value::ValueType::ImplicitPointer: + // The pointed-at object is itself an optimized-out pointer. + return FormatImplicitPointer(stream, pointed_to, *pointee, context, + depth + 1); + + case Value::ValueType::FileAddress: + case Value::ValueType::Invalid: + break; + } + OXCAML_EMIT_MARKER(stream, "", + "Unsupported materialized value kind {0} for an " + "optimized-out pointer of type '{1}'", + static_cast(pointee->GetValueType()), + ptr_type->GetDisplayName()); + return true; } // Format an OCaml exception value @@ -550,8 +772,7 @@ ExtractExceptionInfo(lldb::addr_t exception_addr, lldb::ProcessSP process_sp) { // Format an OCaml exception value static bool FormatException(Stream &stream, DataExtractor &data, - lldb::ProcessSP process_sp, - const ExecutionContextRef &exe_ctx_ref) { + const OxCamlFormatContext &context) { lldb::offset_t offset = 0; uint64_t exception_value = data.GetU64(&offset); @@ -565,7 +786,7 @@ static bool FormatException(Stream &stream, DataExtractor &data, exception_value, (exception_value == 0) ? "null pointer" : "immediate OCaml value"); - auto info_opt = ExtractExceptionInfo(exception_value, process_sp); + auto info_opt = ExtractExceptionInfo(exception_value, context.process_sp); ENSURE(info_opt.has_value(), stream, "", "Failed to extract OCaml exception payload at 0x{0:x}", @@ -583,8 +804,9 @@ static bool FormatException(Stream &stream, DataExtractor &data, } DataExtractor arg_data(&info_opt->arguments[i], constants::WORD_SIZE, - process_sp->GetByteOrder(), constants::WORD_SIZE); - oxcaml::FormatOxCamlValue(stream, arg_data, process_sp, exe_ctx_ref); + context.byte_order, constants::WORD_SIZE); + oxcaml::FormatOxCamlValue(stream, arg_data, context.process_sp, + context.exe_ctx_ref); } if (use_parens) { @@ -595,161 +817,356 @@ static bool FormatException(Stream &stream, DataExtractor &data, return true; } -static bool FormatArray(Stream &stream, OxCamlArrayType *array_type, - DataExtractor &data, lldb::ProcessSP process_sp, - const ExecutionContextRef &exe_ctx_ref) { - ENSURE(data.GetByteSize() >= helpers::constants::WORD_SIZE, stream, "", - "Data size {0} < expected word size {1}", data.GetByteSize(), - helpers::constants::WORD_SIZE); - - lldb::offset_t offset = 0; - uint64_t array_ptr = data.GetU64(&offset); - - ENSURE(offset != 0, stream, "", - "Failed to read OCaml array pointer from value data ({0} bytes " - "available)", - data.GetByteSize()); - - // Check for null or immediate values (not actual pointers) - ENSURE(array_ptr != 0 && !helpers::value::IsImmediate(array_ptr), stream, - llvm::formatv("", array_ptr).str(), - "Array pointer 0x{0:x} is a {1}; skipping dereference", array_ptr, - array_ptr == 0 ? "null pointer" : "immediate OCaml value"); - +// Polymorphic array values such as ['a array] may have an ocaml_value element +// type in DWARF but actually carry unboxed doubles at runtime. The OCaml block +// tag (Double_array_tag) is authoritative, so check it here before formatting +// elements from DWARF. Returns true (with output emitted) if the runtime tag +// proves this is a float array; returns false without emitting anything +// otherwise. +static bool TryFormatRuntimeFloatArray(Stream &stream, lldb::addr_t array_ptr, + lldb::ProcessSP process_sp) { auto header_opt = helpers::ReadBlockHeader(array_ptr, process_sp); - ENSURE(header_opt.has_value(), stream, - llvm::formatv("", array_ptr).str(), - "Failed to read OCaml array header for pointer 0x{0:x}", array_ptr); + if (!header_opt.has_value()) + return false; - uint64_t header = *header_opt; uint8_t tag; uint64_t wosize; uint8_t reserved; - helpers::header::ParseHeader(header, tag, wosize, reserved); + helpers::header::ParseHeader(*header_opt, tag, wosize, reserved); + if (tag != helpers::constants::DOUBLE_ARRAY_TAG) + return false; + + return oxcaml::FormatOxCamlDoubleArray(stream, array_ptr, wosize, process_sp); +} + +static bool FormatArray(Stream &stream, OxCamlArrayType *array_type, + DataExtractor &data, const OxCamlFormatContext &context, + std::optional object_address) { + if (object_address.has_value() && + TryFormatRuntimeFloatArray(stream, *object_address, context.process_sp)) + return true; - // OCaml float arrays (tag 254) store unboxed 8-byte IEEE 754 doubles - // wosize is the number of doubles (1 word = 1 double on 64-bit platforms) - if (tag == constants::DOUBLE_ARRAY_TAG) { - return oxcaml::FormatOxCamlDoubleArray(stream, array_ptr, wosize, - process_sp); + llvm::Expected stride_result = + array_type->GetStride().Evaluate(context, object_address); + if (!stride_result) { + OXCAML_EMIT_MARKER( + stream, "", + "Failed to evaluate byte_stride for array '{0}' (DIE 0x{1:x}): {2}", + array_type->GetDisplayName(), array_type->GetDieId(), + llvm::toString(stride_result.takeError())); + return true; } + uint64_t stride = stride_result->GetScalar(); + ENSURE(stride != 0, stream, "", + "Array type '{0}' (DIE 0x{1:x}) has stride 0; cannot format elements", + array_type->GetDisplayName(), array_type->GetDieId()); + + // OxCaml compiler-emitted array DIEs always carry DW_AT_count on their + // subrange. Values without typed-array DWARF are handled by generic + // ocaml_value formatting instead. + ENSURE(array_type->GetCount().has_value(), stream, "", + "Array type '{0}' (DIE 0x{1:x}) has no DW_AT_count; " + "compiler-emitted OxCaml arrays must include a subrange count " + "expression", + array_type->GetDisplayName(), array_type->GetDieId()); + llvm::Expected count_result = + array_type->GetCount()->Evaluate(context, object_address); + if (!count_result) { + OXCAML_EMIT_MARKER( + stream, "", + "Failed to evaluate DW_AT_count for array '{0}' (DIE 0x{1:x}): {2}", + array_type->GetDisplayName(), array_type->GetDieId(), + llvm::toString(count_result.takeError())); + return true; + } + uint64_t num_elements = count_result->GetScalar(); - // Regular arrays: calculate number of elements from wosize and stride - // wosize is number of words, total bytes = wosize * WORD_SIZE - OxCamlType *element_type = array_type->GetElementType(); - uint64_t stride = array_type->GetStride(); - ENSURE(stride != 0, stream, llvm::formatv("", array_ptr).str(), - "Array type '{0}' (DIE 0x{1:x}) has stride 0; cannot format array at " - "0x{2:x}", - array_type->GetDisplayName(), array_type->GetDieId(), array_ptr); - - uint64_t total_bytes = wosize * helpers::constants::WORD_SIZE; - uint64_t num_elements = total_bytes / stride; - Status error; + uint64_t payload_size = num_elements * stride; + ENSURE(data.GetByteSize() >= payload_size, stream, "", + "Array '{0}' (DIE 0x{1:x}) payload buffer holds {2} bytes, needs " + "{3} ({4} elements x {5}-byte stride)", + array_type->GetDisplayName(), array_type->GetDieId(), + data.GetByteSize(), payload_size, num_elements, stride); + OxCamlType *element_type = array_type->GetElementType(); stream.Printf("[| "); - for (uint64_t i = 0; i < num_elements; i++) { if (i > 0) stream.Printf("; "); - uint64_t element_address = array_ptr + (i * stride); + DataExtractor element_data(data, i * stride, stride); + std::optional element_address; + if (object_address.has_value()) + element_address = *object_address + i * stride; + FormatValue(stream, element_type, element_data, context, element_address); + } + stream.Printf(" |]"); + return true; +} - std::vector buffer(stride); - size_t bytes_read = - process_sp->ReadMemory(element_address, buffer.data(), stride, error); +// A member's location resolved against the parent's object address and the +// parent's in-buffer data: either an offset within [data] (for constant +// object-relative locations) or a process load address (for exprloc results). +// The member object address is the OCaml object address of the member itself, +// suitable for evaluating expressions on the member's type. +struct ResolvedMemberLocation { + enum class Source { BufferOffset, LoadAddress }; + Source source; + uint64_t buffer_offset = 0; + lldb::addr_t load_address = LLDB_INVALID_ADDRESS; + std::optional member_object_address; +}; - if (bytes_read != stride || error.Fail()) { - OXCAML_EMIT_MARKER(stream, "", - "Failed to read array element {0} at 0x{1:x}; " - "requested {2} bytes, read {3}", - i, element_address, stride, bytes_read); - error.Clear(); - continue; - } +static std::optional +ResolveMemberLocation(Stream &stream, const OxCamlMember &member, + DataExtractor &data, const OxCamlFormatContext &context, + std::optional parent_object_address) { + (void)data; + llvm::Expected result = + member.location.Evaluate(context, parent_object_address); + if (!result) { + OXCAML_EMIT_MARKER( + stream, "", "Failed to evaluate location for member '{0}': {1}", + member.name.value_or(""), llvm::toString(result.takeError())); + return std::nullopt; + } - DataExtractor element_data(buffer.data(), stride, data.GetByteOrder(), - data.GetAddressByteSize()); - FormatValue(stream, element_type, element_data, process_sp, exe_ctx_ref); + ResolvedMemberLocation resolved; + if (result->GetLocationStorage() == + OxCamlLayoutResult::LocationStorage::ObjectRelativeByteOffset) { + resolved.source = ResolvedMemberLocation::Source::BufferOffset; + resolved.buffer_offset = result->GetObjectRelativeByteOffset(); + if (parent_object_address.has_value()) + resolved.member_object_address = + *parent_object_address + resolved.buffer_offset; + } else { + resolved.source = ResolvedMemberLocation::Source::LoadAddress; + resolved.load_address = result->GetLoadAddress(); + resolved.member_object_address = resolved.load_address; } + return resolved; +} - stream.Printf(" |]"); +// Read [byte_size] bytes for a member or discriminator into [out_buffer] +// according to a resolved location. +// +// [implicit_pieces] is the implicit-pointer table of [data]'s composite (if +// any): ranges listed there contain placeholder bytes rather than data, so +// reads overlapping them are refused, except that pieces falling entirely +// inside the read are rebased onto the copied bytes and appended to +// [contained_pieces] when the caller provides it (the caller then interprets +// them against the copy). +static bool ReadResolvedMemberBytes( + Stream &stream, const OxCamlMember &member, uint64_t byte_size, + const ResolvedMemberLocation &resolved, DataExtractor &data, + const OxCamlFormatContext &context, + llvm::SmallVectorImpl &out_buffer, + llvm::ArrayRef implicit_pieces = {}, + llvm::SmallVectorImpl *contained_pieces = + nullptr) { + out_buffer.resize(byte_size); + if (resolved.source == ResolvedMemberLocation::Source::BufferOffset) { + uint64_t offset = resolved.buffer_offset; + for (const Value::ImplicitPointerPiece &piece : implicit_pieces) { + const bool overlaps = piece.buffer_offset < offset + byte_size && + offset < piece.buffer_offset + piece.byte_size; + if (!overlaps) + continue; + const bool contained = + piece.buffer_offset >= offset && + piece.buffer_offset + piece.byte_size <= offset + byte_size; + if (contained && contained_pieces) { + contained_pieces->push_back( + {piece.buffer_offset - offset, piece.byte_size, piece.pointee}); + continue; + } + // Refuse rather than reading the placeholder bytes: an optimized-out + // pointer's value must never be conflated with bytes such as zeros. + OXCAML_EMIT_MARKER( + stream, "", + "Member '{0}' overlaps an optimized-out (implicit pointer) range " + "of its parent that cannot be read as bytes", + member.name.value_or("")); + return false; + } + if (offset + byte_size > data.GetByteSize()) { + OXCAML_EMIT_MARKER( + stream, "", + "Member '{0}' at object-relative offset {1} (+{2} bytes) extends " + "past available data ({3} bytes)", + member.name.value_or(""), offset, byte_size, + data.GetByteSize()); + return false; + } + if (data.CopyData(offset, byte_size, out_buffer.data()) != byte_size) { + OXCAML_EMIT_MARKER( + stream, "", + "Member '{0}': failed to copy {1} bytes from offset {2}", + member.name.value_or(""), byte_size, offset); + return false; + } + return true; + } + + Status error; + size_t bytes_read = context.process_sp->ReadMemory( + resolved.load_address, out_buffer.data(), byte_size, error); + if (bytes_read != byte_size || error.Fail()) { + OXCAML_EMIT_MARKER( + stream, "", + "Member '{0}': failed to read {1} bytes from 0x{2:x}: {3}", + member.name.value_or(""), byte_size, resolved.load_address, + error.AsCString()); + return false; + } return true; } static bool FormatMember(Stream &stream, const OxCamlMember &member, - DataExtractor &data, lldb::ProcessSP process_sp, - const ExecutionContextRef &exe_ctx_ref) { + DataExtractor &data, + const OxCamlFormatContext &context, + std::optional object_address, + llvm::ArrayRef + implicit_pieces = {}) { + auto resolved = + ResolveMemberLocation(stream, member, data, context, object_address); + if (!resolved.has_value()) + return true; + if (member.IsBitField()) { - uint64_t bit_offset = member.bit_offset.value(); - uint64_t bit_size = member.bit_size.value(); - lldb::offset_t byte_offset = member.data_member_location; + llvm::Expected bit_offset_result = + member.bit_offset->Evaluate(context, object_address); + if (!bit_offset_result) { + OXCAML_EMIT_MARKER(stream, "", + "Failed to evaluate bit offset for member '{0}': {1}", + member.name.value_or(""), + llvm::toString(bit_offset_result.takeError())); + return true; + } + llvm::Expected bit_size_result = + member.bit_size->Evaluate(context, object_address); + if (!bit_size_result) { + OXCAML_EMIT_MARKER(stream, "", + "Failed to evaluate bit size for member '{0}': {1}", + member.name.value_or(""), + llvm::toString(bit_size_result.takeError())); + return true; + } + uint64_t bit_offset = bit_offset_result->GetScalar(); + uint64_t bit_size = bit_size_result->GetScalar(); - // Check if bit-field spans multiple words (not currently supported) - // Multi-word bit-fields would require reading multiple 64-bit words and - // combining bits across word boundaries ENSURE(bit_offset + bit_size <= 64, stream, "", "Bit-field '{0}' spans multiple words (offset={1}, size={2}); " "multi-word bit-fields not yet supported", member.name.value_or(""), bit_offset, bit_size); - ENSURE(byte_offset + sizeof(uint64_t) <= data.GetByteSize(), stream, - "", - "Not enough data ({0} bytes) to read bit-field '{1}' starting at " - "offset {2}", - data.GetByteSize(), member.name.value_or(""), - static_cast(byte_offset)); + llvm::SmallVector word; + if (!ReadResolvedMemberBytes(stream, member, sizeof(uint64_t), *resolved, + data, context, word, implicit_pieces)) + return true; - uint64_t full_value = data.GetU64(&byte_offset); + DataExtractor word_data(word.data(), word.size(), data.GetByteOrder(), + data.GetAddressByteSize()); + lldb::offset_t off = 0; + uint64_t full_value = word_data.GetU64(&off); uint64_t bit_mask = MakeLowBitMask(bit_size); uint64_t extracted = (full_value >> bit_offset) & bit_mask; + // The extracted bit-field value is a fresh scalar with no header; nested + // expressions cannot reference an object address through it. DataExtractor bit_data(&extracted, sizeof(extracted), data.GetByteOrder(), data.GetAddressByteSize()); + return FormatValue(stream, member.GetType(), bit_data, context, + /*object_address=*/std::nullopt); + } - return FormatValue(stream, member.GetType(), bit_data, process_sp, - exe_ctx_ref); + // Resolve the member type's dynamic byte size using the member's own + // object address, so nested structures with exprloc-valued sizes read the + // right amount of memory. + llvm::Expected member_byte_size_expected = ResolveTypeByteSize( + member.GetType(), resolved->member_object_address, context); + if (!member_byte_size_expected) { + OXCAML_EMIT_MARKER( + stream, "", + "Failed to evaluate byte size for member '{0}' (type '{1}'): {2}", + member.name.value_or(""), member.GetType()->GetDisplayName(), + llvm::toString(member_byte_size_expected.takeError())); + return true; + } + uint64_t member_byte_size = *member_byte_size_expected; + + // The member's bytes may not exist at all: if its range is exactly an + // implicit-pointer piece of the parent's composite, the member is an + // optimized-out pointer whose pointee is described by another DIE. + if (resolved->source == ResolvedMemberLocation::Source::BufferOffset) { + for (const Value::ImplicitPointerPiece &piece : implicit_pieces) { + if (piece.buffer_offset == resolved->buffer_offset && + piece.byte_size == member_byte_size) { + Value pointer; + pointer.SetImplicitPointer(piece.pointee.delegate, + piece.pointee.die_offset, + piece.pointee.byte_offset); + return FormatImplicitPointer(stream, member.GetType(), pointer, + context); + } + } } - DataExtractor member_data(data, member.data_member_location, - member.GetType()->GetByteSize()); - return FormatValue(stream, member.GetType(), member_data, process_sp, - exe_ctx_ref); + llvm::SmallVector buffer; + llvm::SmallVector member_pieces; + if (!ReadResolvedMemberBytes(stream, member, member_byte_size, *resolved, + data, context, buffer, implicit_pieces, + &member_pieces)) + return true; + + DataExtractor member_data(buffer.data(), member_byte_size, + data.GetByteOrder(), data.GetAddressByteSize()); + return FormatValue(stream, member.GetType(), member_data, context, + resolved->member_object_address, member_pieces); +} + +// Returns true if both members have constant data_member_location 0. +static bool AreBothMembersAtConstantOffsetZero(const OxCamlMember &a, + const OxCamlMember &b) { + return a.location.IsConstant() && a.location.GetConstant() == 0 && + b.location.IsConstant() && b.location.GetConstant() == 0; } static bool FormatStructure(Stream &stream, OxCamlStructureType *struct_type, - DataExtractor &data, lldb::ProcessSP process_sp, - const ExecutionContextRef &exe_ctx_ref) { + DataExtractor &data, + const OxCamlFormatContext &context, + std::optional object_address, + llvm::ArrayRef + implicit_pieces) { const auto &members = struct_type->GetMembers(); const auto &variant_parts = struct_type->GetVariantParts(); - // Special case for OCaml variants: Structures with no direct members and - // exactly one variant part are formatted without braces (just the variant - // content directly). This represents the typical OCaml variant structure - // encoding. + // OCaml variant: a structure with no direct members and exactly one + // variant part is rendered as the variant content alone (no braces). if (members.empty() && variant_parts.size() == 1) { - return FormatVariantPart(stream, variant_parts[0], data, process_sp, - exe_ctx_ref, true); + return FormatVariantPart(stream, variant_parts[0], data, context, + object_address, true, implicit_pieces); } - // Special case for OCaml exceptions: Union-style structure with "exn" and - // "raw" members. Exceptions have both members at the same offset. We format - // only the "raw" member. This hardcodes the current format of the OxCaml - // compiler. + // OCaml exception: a union-style struct with "exn" and "raw" members both + // at offset 0. Format only via FormatException. if (members.size() == 2 && variant_parts.empty() && members[0].name.has_value() && members[1].name.has_value() && - members[0].data_member_location == 0 && - members[1].data_member_location == 0) { + AreBothMembersAtConstantOffsetZero(members[0], members[1])) { const std::string &name0 = members[0].name.value(); const std::string &name1 = members[1].name.value(); if ((name0 == "exn" && name1 == "raw") || (name0 == "raw" && name1 == "exn")) { - return FormatException(stream, data, process_sp, exe_ctx_ref); + // FormatException reads raw bytes itself and cannot interpret + // optimized-out (implicit pointer) ranges. + ENSURE(implicit_pieces.empty(), stream, "", + "Exception value contains optimized-out (implicit pointer) " + "ranges{0}", + ""); + return FormatException(stream, data, context); } } - // Regular case: structure with members and/or multiple variant parts - // Determine formatting based on type bool is_tuple = struct_type->IsTuple(); const char *open_delim = is_tuple ? "(" : "{ "; const char *close_delim = is_tuple ? ")" : " }"; @@ -759,7 +1176,6 @@ static bool FormatStructure(Stream &stream, OxCamlStructureType *struct_type, bool has_content = false; - // Format regular members first for (size_t i = 0; i < members.size(); ++i) { if (has_content) stream.Printf("%s", separator); @@ -768,17 +1184,17 @@ static bool FormatStructure(Stream &stream, OxCamlStructureType *struct_type, stream.Printf("%s = ", members[i].name.value().c_str()); } - FormatMember(stream, members[i], data, process_sp, exe_ctx_ref); + FormatMember(stream, members[i], data, context, object_address, + implicit_pieces); has_content = true; } - // Format variant parts for (size_t i = 0; i < variant_parts.size(); ++i) { if (has_content) stream.Printf("%s", separator); - FormatVariantPart(stream, variant_parts[i], data, process_sp, exe_ctx_ref, - false); + FormatVariantPart(stream, variant_parts[i], data, context, object_address, + false, implicit_pieces); has_content = true; } @@ -788,118 +1204,118 @@ static bool FormatStructure(Stream &stream, OxCamlStructureType *struct_type, } static bool FormatValue(Stream &stream, OxCamlType *type, DataExtractor &data, - lldb::ProcessSP process_sp, - const ExecutionContextRef &exe_ctx_ref) { + const OxCamlFormatContext &context, + std::optional object_address, + llvm::ArrayRef + implicit_pieces) { + // Only the aggregate formatters below know how to interpret buffers that + // contain optimized-out (implicit pointer) ranges; refuse everywhere else + // rather than reading their placeholder bytes. + if (!implicit_pieces.empty() && + (!type || (type->GetKind() != OxCamlType::Typedef && + type->GetKind() != OxCamlType::Structure))) { + OXCAML_EMIT_MARKER(stream, "", + "Value of type '{0}' contains optimized-out (implicit " + "pointer) ranges, which this formatter cannot render", + type ? type->GetDisplayName() : ""); + return true; + } + if (!type) { Log *log = GetLog(OxCamlLog::Formatting); LLDB_LOG( log, "FormatValue: No type information available, using fallback formatter"); - return FormatFallback(stream, type, data, process_sp); + return FormatFallback(stream, type, data); } switch (type->GetKind()) { case OxCamlType::Value: - return oxcaml::FormatOxCamlValue(stream, data, process_sp, exe_ctx_ref); + return oxcaml::FormatOxCamlValue(stream, data, context.process_sp, + context.exe_ctx_ref); case OxCamlType::UnboxedBase: return FormatUnboxedBase(stream, static_cast(type), - data, process_sp); + data); case OxCamlType::Enum: - return FormatEnum(stream, static_cast(type), data, - process_sp); + return FormatEnum(stream, static_cast(type), data); case OxCamlType::Pointer: return FormatPointer(stream, static_cast(type), data, - process_sp, exe_ctx_ref); + context); case OxCamlType::Typedef: return FormatTypedef(stream, static_cast(type), data, - process_sp, exe_ctx_ref); + context, object_address, implicit_pieces); case OxCamlType::Structure: return FormatStructure(stream, static_cast(type), - data, process_sp, exe_ctx_ref); + data, context, object_address, implicit_pieces); case OxCamlType::Array: return FormatArray(stream, static_cast(type), data, - process_sp, exe_ctx_ref); + context, object_address); case OxCamlType::Placeholder: return FormatPlaceholder(stream, static_cast(type), - data, process_sp); + data); case OxCamlType::Unknown: - return FormatUnknown(stream, static_cast(type), data, - process_sp); + return FormatUnknown(stream, static_cast(type), data); } return false; } -// ============================================================================ -// Variant Formatting -// ============================================================================ -// -// ## Problem Overview -// -// OCaml variants present a formatting challenge: the actual memory size of a -// variant value depends on which constructor is active at runtime. In the -// current implementation of the OxCaml compiler and the LLDB plugin, the DWARF -// information only provides static size information (the base structure size), -// but each variant constructor may have a different payload size. -// -// ## Current Approach: Approximate Two-Pass Strategy -// -// Since the DWARF doesn't encode dynamic size calculations (which would require -// DWARF expressions), we use an estimation approach: -// -// 1. **First Pass - Read Discriminators**: -// - Calculate the minimum memory needed to read all discriminators -// - Read enough memory to analyze which variant constructors are active -// -// 2. **Second Pass - Estimate Total Size**: -// - Based on active variants, calculate the maximum offset required -// - This includes: base structure members + active variant member payloads -// - Read the estimated total memory needed -// -// ## Key Functions: -// -// - `ReadDiscriminatorValue()` - Extracts discriminator value from data -// - `CalculateMinimumSizeForDiscriminators()` - Computes memory needed for -// discriminator analysis -// - `FindActiveVariantsInStructure()` - Determines which variants are active -// based on discriminator values -// - `EstimatePointerAllocationSize()` - Estimates total allocation size by -// examining all active variant payloads -// - `FormatVariantPart()` - Main dispatcher for variant formatting -// - `FormatVariantPartOxCaml()` - OCaml-style variant formatting -// - `FormatVariantPartGeneric()` - Generic variant formatting -// -// ## Limitations -// -// This approach provides an estimate rather than exact sizes: -// - We calculate the maximum offset across all members in the active variant -// - Member type sizes may themselves be estimates (e.g., nested structures) -// - The actual memory layout might be more compact than our calculation -// suggests -// -// A more precise solution would require the OCaml compiler to emit DWARF -// expressions that dynamically calculate exact sizes based on runtime values. -// -// ============================================================================ - static std::optional -ReadDiscriminatorValue(const OxCamlVariantPart &variant_part, - DataExtractor &data) { +ReadDiscriminatorValue(Stream &stream, const OxCamlVariantPart &variant_part, + DataExtractor &data, const OxCamlFormatContext &context, + std::optional object_address, + llvm::ArrayRef + implicit_pieces = {}) { const auto &discriminator = variant_part.GetDiscriminator(); - lldb::offset_t discr_offset = discriminator.data_member_location; - uint64_t discriminator_byte_size = discriminator.GetType()->GetByteSize(); + // The discriminator value is later returned as a uint64_t, so wider + // discriminators cannot fit and we refuse to read them here. + if (discriminator_byte_size == 0 || + discriminator_byte_size > sizeof(uint64_t)) + return std::nullopt; + + auto resolved = ResolveMemberLocation(stream, discriminator, data, context, + object_address); + if (!resolved.has_value()) + return std::nullopt; + llvm::SmallVector buffer; + if (!ReadResolvedMemberBytes(stream, discriminator, discriminator_byte_size, + *resolved, data, context, buffer, + implicit_pieces)) + return std::nullopt; + + DataExtractor discr_data(buffer.data(), buffer.size(), data.GetByteOrder(), + data.GetAddressByteSize()); + lldb::offset_t cursor = 0; std::optional apint = - helpers::ExtractAPInt(data, &discr_offset, discriminator_byte_size); - if (!apint.has_value() || apint->getBitWidth() > 64) { + helpers::ExtractAPInt(discr_data, &cursor, discriminator_byte_size); + if (!apint.has_value() || apint->getBitWidth() > 64) return std::nullopt; - } uint64_t value = apint->getZExtValue(); if (discriminator.IsBitField()) { - uint64_t bit_offset = discriminator.bit_offset.value(); - uint64_t bit_size = discriminator.bit_size.value(); + Log *log = GetLog(OxCamlLog::Formatting); + auto bit_offset_result = + discriminator.bit_offset->Evaluate(context, object_address); + if (!bit_offset_result) { + LLDB_LOG_ERROR(log, bit_offset_result.takeError(), + "ReadDiscriminatorValue: bit_offset evaluation failed for " + "discriminator '{1}': {0}", + discriminator.name); + return std::nullopt; + } + auto bit_size_result = + discriminator.bit_size->Evaluate(context, object_address); + if (!bit_size_result) { + LLDB_LOG_ERROR(log, bit_size_result.takeError(), + "ReadDiscriminatorValue: bit_size evaluation failed for " + "discriminator '{1}': {0}", + discriminator.name); + return std::nullopt; + } + uint64_t bit_offset = bit_offset_result->GetScalar(); + uint64_t bit_size = bit_size_result->GetScalar(); if (bit_offset + bit_size > 64) return std::nullopt; uint64_t discr_mask = MakeLowBitMask(bit_size); @@ -909,127 +1325,6 @@ ReadDiscriminatorValue(const OxCamlVariantPart &variant_part, return value; } -static uint64_t -CalculateMinimumSizeForDiscriminators(OxCamlStructureType *struct_type) { - uint64_t max_discriminator_end = 0; - - const auto &variant_parts = struct_type->GetVariantParts(); - for (const auto &variant_part : variant_parts) { - const auto &discr = variant_part.GetDiscriminator(); - uint64_t discr_end = - discr.data_member_location + discr.GetType()->GetByteSize(); - max_discriminator_end = std::max(max_discriminator_end, discr_end); - } - - return max_discriminator_end; -} - -static std::vector -FindActiveVariantsInStructure(OxCamlStructureType *struct_type, - DataExtractor &data) { - std::vector active_variants; - Log *log = GetLog(OxCamlLog::UserVisibleErrors); - - const auto &variant_parts = struct_type->GetVariantParts(); - for (const auto &variant_part : variant_parts) { - auto discr_value_opt = ReadDiscriminatorValue(variant_part, data); - if (!discr_value_opt.has_value()) { - LLDB_LOG(log, - "Failed to read discriminator value for variant part with " - "discriminator '{0}'", - variant_part.GetDiscriminator().name); - continue; // Skip variant part if discriminator cannot be read - } - - auto active_variant = variant_part.GetActiveVariant(*discr_value_opt); - if (active_variant.has_value()) { - active_variants.push_back(*active_variant); - } - } - - return active_variants; -} - -static uint64_t EstimatePointerAllocationSize(OxCamlType *type, - DataExtractor &data) { - OX_ASSERT(type != nullptr, - "EstimatePointerAllocationSize called with null type (ptr={0:P})", - type); - - while (type->GetKind() == OxCamlType::Typedef) { - type = static_cast(type)->GetUnderlyingType(); - } - - // We assume all types except for structures return accurate sizes. Only - // structures can have variant parts. - if (type->GetKind() != OxCamlType::Structure) { - return type->GetByteSize(); - } - - auto *struct_type = static_cast(type); - uint64_t max_end_offset = type->GetByteSize(); - - const auto &members = struct_type->GetMembers(); - for (const auto &member : members) { - // If the size that is returned here is not exact (e.g., as in the case of - // structures, then our estimate here is off). That's why it is not exact. - uint64_t member_end = - member.data_member_location + member.GetType()->GetByteSize(); - max_end_offset = std::max(max_end_offset, member_end); - } - - auto active_variants = FindActiveVariantsInStructure(struct_type, data); - for (const auto *variant : active_variants) { - for (const auto &member : variant->members) { - uint64_t member_end = - member.data_member_location + member.GetType()->GetByteSize(); - max_end_offset = std::max(max_end_offset, member_end); - } - } - - return max_end_offset; -} - -static uint64_t ComputeActualPointerSize(OxCamlType *pointed_to, - lldb::addr_t adjusted_address, - const DataExtractor &data, - lldb::ProcessSP process_sp) { - uint64_t type_size = pointed_to->GetByteSize(); - - if (pointed_to->GetKind() != OxCamlType::Structure) { - return type_size; - } - - auto *struct_type = static_cast(pointed_to); - const auto &variant_parts = struct_type->GetVariantParts(); - - if (variant_parts.empty()) { - return type_size; - } - - uint64_t min_discriminator_size = - CalculateMinimumSizeForDiscriminators(struct_type); - uint64_t temp_read_size = std::max(min_discriminator_size, type_size); - - // Two-pass approach for structures with variant parts: - // 1. Read enough data to analyze all discriminators - // 2. Calculate precise size based on active variants - Status error; - std::vector temp_buffer(temp_read_size); - size_t bytes_read = process_sp->ReadMemory( - adjusted_address, temp_buffer.data(), temp_buffer.size(), error); - - if (bytes_read >= min_discriminator_size) { - DataExtractor heap_data(temp_buffer.data(), bytes_read, data.GetByteOrder(), - data.GetAddressByteSize()); - uint64_t estimated_size = - EstimatePointerAllocationSize(pointed_to, heap_data); - return estimated_size; - } - - return type_size; -} - enum VariantKind { SingleEntryVariant, // 1 member, no name TupleVariant, // >1 members, no names @@ -1038,10 +1333,15 @@ enum VariantKind { // Format variant part with generic square bracket format: // Name[mem1 = val1; ...; memN = valN] -static bool FormatVariantPartGeneric( - Stream &stream, const OxCamlVariantPart &variant_part, DataExtractor &data, - lldb::ProcessSP process_sp, const ExecutionContextRef &exe_ctx_ref, - uint64_t discr_value, const std::vector &members) { +static bool FormatVariantPartGeneric(Stream &stream, + const OxCamlVariantPart &variant_part, + DataExtractor &data, + const OxCamlFormatContext &context, + std::optional object_address, + uint64_t discr_value, + const std::vector &members, + llvm::ArrayRef + implicit_pieces) { std::string discr_name = "Unknown"; const auto &discriminator = variant_part.GetDiscriminator(); if (discriminator.GetType()->GetKind() == OxCamlType::Enum) { @@ -1062,7 +1362,8 @@ static bool FormatVariantPartGeneric( stream.Printf("; "); if (members[i].name.has_value()) stream.Printf("%s = ", members[i].name.value().c_str()); - FormatMember(stream, members[i], data, process_sp, exe_ctx_ref); + FormatMember(stream, members[i], data, context, object_address, + implicit_pieces); } stream.Printf("]"); } @@ -1070,10 +1371,15 @@ static bool FormatVariantPartGeneric( return true; } -static bool FormatVariantPartOxCaml( - Stream &stream, const OxCamlVariantPart &variant_part, DataExtractor &data, - lldb::ProcessSP process_sp, const ExecutionContextRef &exe_ctx_ref, - uint64_t discr_value, const std::vector &members) { +static bool FormatVariantPartOxCaml(Stream &stream, + const OxCamlVariantPart &variant_part, + DataExtractor &data, + const OxCamlFormatContext &context, + std::optional object_address, + uint64_t discr_value, + const std::vector &members, + llvm::ArrayRef + implicit_pieces) { const auto &discriminator = variant_part.GetDiscriminator(); ENSURE(discriminator.GetType()->GetKind() == OxCamlType::Enum, stream, @@ -1150,7 +1456,8 @@ static bool FormatVariantPartOxCaml( } } - FormatMember(stream, members[i], data, process_sp, exe_ctx_ref); + FormatMember(stream, members[i], data, context, object_address, + implicit_pieces); } stream.Printf("%s", close_delim); @@ -1158,18 +1465,19 @@ static bool FormatVariantPartOxCaml( return true; } -// Main variant formatting dispatcher -// -// Special handling for artificial discriminators: -// - Artificial discriminators (e.g., Pointer/Immediate) with exactly one member -// in the active variant are displayed transparently (member content only) -// - All other cases dispatch to either OCaml or generic formatting +// An artificial discriminator (e.g. Pointer/Immediate) with exactly one +// member in the active variant is displayed transparently (member content +// only, no discriminator name or brackets). static bool FormatVariantPart(Stream &stream, const OxCamlVariantPart &variant_part, - DataExtractor &data, lldb::ProcessSP process_sp, - const ExecutionContextRef &exe_ctx_ref, - bool is_ocaml_variant = false) { - auto discr_value_opt = ReadDiscriminatorValue(variant_part, data); + DataExtractor &data, + const OxCamlFormatContext &context, + std::optional object_address, + bool is_ocaml_variant, + llvm::ArrayRef + implicit_pieces) { + auto discr_value_opt = ReadDiscriminatorValue( + stream, variant_part, data, context, object_address, implicit_pieces); ENSURE(discr_value_opt.has_value(), stream, "", "Discriminator value unreadable (variant={0})", variant_part.GetDiscriminator().name); @@ -1183,18 +1491,19 @@ static bool FormatVariantPart(Stream &stream, const auto &members = (*active_variant)->members; - // Special case: artificial discriminator with exactly one member - // Display the member content directly without discriminator name/brackets if (variant_part.HasArtificialDiscriminator() && members.size() == 1) { - FormatMember(stream, members[0], data, process_sp, exe_ctx_ref); + FormatMember(stream, members[0], data, context, object_address, + implicit_pieces); return true; } if (is_ocaml_variant) { - return FormatVariantPartOxCaml(stream, variant_part, data, process_sp, - exe_ctx_ref, discr_value, members); + return FormatVariantPartOxCaml(stream, variant_part, data, context, + object_address, discr_value, members, + implicit_pieces); } else { - return FormatVariantPartGeneric(stream, variant_part, data, process_sp, - exe_ctx_ref, discr_value, members); + return FormatVariantPartGeneric(stream, variant_part, data, context, + object_address, discr_value, members, + implicit_pieces); } } diff --git a/lldb/source/Plugins/Language/OxCaml/OxCamlLanguage.cpp b/lldb/source/Plugins/Language/OxCaml/OxCamlLanguage.cpp index 102358c3c7aa6..8293b83ad7a57 100644 --- a/lldb/source/Plugins/Language/OxCaml/OxCamlLanguage.cpp +++ b/lldb/source/Plugins/Language/OxCaml/OxCamlLanguage.cpp @@ -8,10 +8,13 @@ #include "OxCamlLanguage.h" #include "LogChannelOxCaml.h" +#include "OxCamlExternalPrinter.h" #include "OxCamlFormatters.h" #include "lldb/Core/PluginManager.h" +#include "lldb/Core/UserSettingsController.h" #include "lldb/DataFormatters/DataVisualization.h" #include "lldb/DataFormatters/TypeSummary.h" +#include "lldb/Interpreter/OptionValueProperties.h" #include "lldb/Target/Language.h" #include "lldb/Utility/ConstString.h" #include "lldb/Utility/Log.h" @@ -20,6 +23,36 @@ using namespace lldb; using namespace lldb_private; +#define LLDB_PROPERTIES_language_oxcaml +#include "LanguageOxCamlProperties.inc" + +enum { +#define LLDB_PROPERTIES_language_oxcaml +#include "LanguageOxCamlPropertiesEnum.inc" +}; + +namespace { +class PluginProperties : public Properties { +public: + static llvm::StringRef GetSettingName() { return "display"; } + + PluginProperties() { + m_collection_sp = std::make_shared(GetSettingName()); + m_collection_sp->Initialize(g_language_oxcaml_properties); + } + + FileSpec GetExternalSummaryExecutable() const { + return GetPropertyAtIndexAs(ePropertyExternalSummaryExecutable, + {}); + } +}; +} // namespace + +static PluginProperties &GetGlobalPluginProperties() { + static PluginProperties g_settings; + return g_settings; +} + lldb::LanguageType OxCamlLanguage::GetLanguageType() const { return lldb::eLanguageTypeOCaml; } @@ -65,10 +98,30 @@ lldb::TypeCategoryImplSP OxCamlLanguage::GetFormatters() { return g_category; } +llvm::Expected +OxCamlLanguage::GetExternalFormatterInput(ValueObject &valobj, + std::vector &data) { + return formatters::oxcaml::GetOCamlExternalFormatterInput(valobj, data); +} + +FileSpec OxCamlLanguage::GetExternalSummaryExecutable() { + return GetGlobalPluginProperties().GetExternalSummaryExecutable(); +} + +void OxCamlLanguage::DebuggerInitialize(Debugger &debugger) { + if (!PluginManager::GetSettingForOxCamlLanguagePlugin( + debugger, PluginProperties::GetSettingName())) { + PluginManager::CreateSettingForOxCamlLanguagePlugin( + debugger, GetGlobalPluginProperties().GetValueProperties(), + "Properties for the OxCaml language plug-in.", + /*is_global_property=*/true); + } +} + void OxCamlLanguage::Initialize() { LogChannelOxCaml::Initialize(); PluginManager::RegisterPlugin(GetPluginNameStatic(), "OxCaml Language", - CreateInstance); + CreateInstance, &DebuggerInitialize); } void OxCamlLanguage::Terminate() { diff --git a/lldb/source/Plugins/Language/OxCaml/OxCamlLanguage.h b/lldb/source/Plugins/Language/OxCaml/OxCamlLanguage.h index c34d7af397427..2ddd067b6bf1e 100644 --- a/lldb/source/Plugins/Language/OxCaml/OxCamlLanguage.h +++ b/lldb/source/Plugins/Language/OxCaml/OxCamlLanguage.h @@ -10,6 +10,7 @@ #define LLDB_SOURCE_PLUGINS_LANGUAGE_OXCAML_OXCAMLLANGUAGE_H #include "lldb/Target/Language.h" +#include "lldb/Utility/FileSpec.h" #include "lldb/lldb-private.h" namespace lldb_private { @@ -28,12 +29,23 @@ class OxCamlLanguage : public Language { lldb::TypeCategoryImplSP GetFormatters() override; + llvm::Expected + GetExternalFormatterInput(ValueObject &valobj, + std::vector &data) override; + + /// The plugin.oxcaml.display.external-summary-executable setting: the + /// external pretty-printer used before falling back to the built-in + /// formatter. Empty when not configured. + static FileSpec GetExternalSummaryExecutable(); + static void Initialize(); static void Terminate(); static lldb_private::Language *CreateInstance(lldb::LanguageType language); + static void DebuggerInitialize(Debugger &debugger); + static llvm::StringRef GetPluginNameStatic(); llvm::StringRef GetPluginName() override; diff --git a/lldb/source/Plugins/Language/OxCaml/OxCamlMarshal.cpp b/lldb/source/Plugins/Language/OxCaml/OxCamlMarshal.cpp new file mode 100644 index 0000000000000..464c0e4471386 --- /dev/null +++ b/lldb/source/Plugins/Language/OxCaml/OxCamlMarshal.cpp @@ -0,0 +1,1048 @@ +//===-- OxCamlMarshal.cpp -------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// Marshaller for OCaml values living in a debugged process, producing the +// standard Marshal wire format. Adapted from the OxCaml runtime's +// runtime/extern.c by way of extern_standalone.cpp (in the extern_standalone +// directory of this repository); the runtime code is deliberately followed +// closely, so much of this file is C-style rather than LLVM-style C++. +// +// Differences from extern_standalone.cpp: +// - Every access to the OCaml heap (block headers, fields, string and +// float-array contents) goes through a MarshalMemoryReader instead of +// dereferencing host pointers. The "extern stack" of pending fields +// stores debuggee addresses, not host pointers. A failed read aborts +// marshalling with an error rather than crashing LLDB. +// - The state is allocated per call instead of being a global, making the +// marshaller safe to use from multiple debugger sessions. +// - The total output size is capped (see MarshalOCamlValue); a debugger can +// be pointed at arbitrary words, and a corrupt "block header" could +// otherwise describe a near-infinite object graph. +// - Errors are reported as llvm::Error. Internally setjmp/longjmp is kept, +// mirroring the runtime's raise-through behaviour; the code inside the +// setjmp region is careful to hold no live C++ objects with destructors. +// +// Inherited from extern_standalone.cpp: 64-bit targets only; sharing always +// preserved; no compression; no custom-block serializers (so Int32.t, +// Int64.t, Nativeint.t, Bigarray values are rejected); closures, +// continuations, abstract and mixed blocks are rejected. +// +//===----------------------------------------------------------------------===// + +#include "OxCamlMarshal.h" + +#include "lldb/Target/Process.h" +#include "lldb/Utility/Status.h" + +#include +#include +#include +#include + +using namespace lldb_private; +using namespace lldb_private::formatters::oxcaml; + +// --------------------------------------------------------------------------- +// Memory readers + +bool MarshalMemoryReader::ReadWord(uint64_t addr, uint64_t &word) { + // All supported OxCaml targets share the host's byte order. + return ReadBytes(addr, &word, sizeof(word)); +} + +bool ProcessMemoryReader::ReadBytes(uint64_t addr, void *buf, uint64_t len) { + if (!m_process_sp) + return false; + if (len == 0) + return true; + Status error; + const size_t bytes_read = + m_process_sp->ReadMemory(addr, buf, static_cast(len), error); + return error.Success() && bytes_read == len; +} + +// --------------------------------------------------------------------------- +// The marshaller + +namespace { + +typedef int64_t intnat; +typedef uint64_t uintnat; +typedef size_t asize_t; + +static inline int umul_overflow(uintnat a, uintnat b, uintnat *res) { + return __builtin_mul_overflow(a, b, res); +} + +/* Value representation (from caml/mlvalues.h). A [value] here is a word + read from the debuggee; "pointers" are debuggee addresses and are never + dereferenced directly. */ + +typedef intnat value; +typedef uintnat header_t; +typedef uintnat mlsize_t; +typedef unsigned int tag_t; + +#define Is_null(v) ((v) == 0) + +static inline int Is_long(value x) { return ((x & 1) != 0 || x == 0); } + +static inline int Is_block(value x) { return ((x & 1) == 0 && x != 0); } + +#define Long_val(x) ((x) >> 1) + +/* Headers: 8 reserved bits (nonzero only for mixed blocks), then 46 size + bits, 2 colour bits and 8 tag bits. Header colour bits are ignored. + Make_header uses the colour for out-of-heap blocks (NOT_MARKABLE, i.e. 3) + and zero reserved bits. */ + +#define Tag_hd(hd) ((tag_t)((hd) & 0xFF)) +#define Wosize_hd(hd) ((mlsize_t)(((hd) >> 10) & (((header_t)1 << 46) - 1))) +#define Reserved_hd(hd) ((hd) >> 56) + +#define Make_header(wosize, tag) (((header_t)(wosize) << 10) + (3 << 8) + (tag)) + +#define Forcing_tag 244 +#define Cont_tag 245 +#define Lazy_tag 246 +#define Closure_tag 247 +#define Infix_tag 249 +#define Forward_tag 250 +#define Abstract_tag 251 +#define String_tag 252 +#define Double_tag 253 +#define Double_array_tag 254 +#define Custom_tag 255 + +/* The marshal format (from caml/intext.h) */ + +#define Intext_magic_number_small 0x8495A6BE +#define Intext_magic_number_big 0x8495A6BF + +/* Header format for the "small" model: 20 bytes + 0 "small" magic number + 4 length of marshaled data, in bytes + 8 number of shared blocks + 12 size in words when read on a 32-bit platform + (always 0 here: 32-bit readers are not supported) + 16 size in words when read on a 64-bit platform + The 4 numbers are 32 bits each, in big endian. + + Header format for the "big" model: 32 bytes + 0 "big" magic number + 4 four reserved bytes, currently set to 0 + 8 length of marshaled data, in bytes + 16 number of shared blocks + 24 size in words when read on a 64-bit platform + The 3 numbers are 64 bits each, in big endian. +*/ + +#define MAX_INTEXT_HEADER_SIZE 32 + +/* Codes for the compact format */ + +#define PREFIX_SMALL_BLOCK 0x80 +#define PREFIX_SMALL_INT 0x40 +#define PREFIX_SMALL_STRING 0x20 +#define CODE_INT8 0x0 +#define CODE_INT16 0x1 +#define CODE_INT32 0x2 +#define CODE_INT64 0x3 +#define CODE_SHARED8 0x4 +#define CODE_SHARED16 0x5 +#define CODE_SHARED32 0x6 +#define CODE_SHARED64 0x14 +#define CODE_BLOCK32 0x8 +#define CODE_BLOCK64 0x13 +#define CODE_STRING8 0x9 +#define CODE_STRING32 0xA +#define CODE_STRING64 0x15 +#define CODE_DOUBLE_BIG 0xB +#define CODE_DOUBLE_LITTLE 0xC +#define CODE_DOUBLE_ARRAY8_BIG 0xD +#define CODE_DOUBLE_ARRAY8_LITTLE 0xE +#define CODE_DOUBLE_ARRAY32_BIG 0xF +#define CODE_DOUBLE_ARRAY32_LITTLE 0x7 +#define CODE_DOUBLE_ARRAY64_BIG 0x16 +#define CODE_DOUBLE_ARRAY64_LITTLE 0x17 +#define CODE_NULL 0x1f + +#if defined(__BYTE_ORDER__) && __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ +#define CODE_DOUBLE_NATIVE CODE_DOUBLE_BIG +#define CODE_DOUBLE_ARRAY8_NATIVE CODE_DOUBLE_ARRAY8_BIG +#define CODE_DOUBLE_ARRAY32_NATIVE CODE_DOUBLE_ARRAY32_BIG +#define CODE_DOUBLE_ARRAY64_NATIVE CODE_DOUBLE_ARRAY64_BIG +#define Htobe16(x) ((uint16_t)(x)) +#define Htobe32(x) ((uint32_t)(x)) +#define Htobe64(x) ((uint64_t)(x)) +#else +#define CODE_DOUBLE_NATIVE CODE_DOUBLE_LITTLE +#define CODE_DOUBLE_ARRAY8_NATIVE CODE_DOUBLE_ARRAY8_LITTLE +#define CODE_DOUBLE_ARRAY32_NATIVE CODE_DOUBLE_ARRAY32_LITTLE +#define CODE_DOUBLE_ARRAY64_NATIVE CODE_DOUBLE_ARRAY64_LITTLE +#define Htobe16(x) __builtin_bswap16((uint16_t)(x)) +#define Htobe32(x) __builtin_bswap32((uint32_t)(x)) +#define Htobe64(x) __builtin_bswap64((uint64_t)(x)) +#endif + +/* Size-ing data structures for extern. Chosen so that + sizeof(struct output_block) is slightly below 8Kb. */ + +#define SIZE_EXTERN_OUTPUT_BLOCK 8100 + +struct output_block { + struct output_block *next; + char *end; + char data[SIZE_EXTERN_OUTPUT_BLOCK]; +}; + +/* Error codes. The message carried alongside is what reaches the user. */ + +typedef enum { + EXTERN_OK = 0, + EXTERN_ERROR_OUT_OF_MEMORY = 1, + EXTERN_ERROR_INVALID_ARGUMENT = 2, + EXTERN_ERROR_READ_FAILURE = 3, + EXTERN_ERROR_OUTPUT_TOO_LARGE = 4 +} extern_error_code_t; + +/* Stack for pending values to marshal. Each item is a run of block fields + still to be serialized, identified by the debuggee address of the next + field (the runtime version stores a host pointer instead). */ + +#define EXTERN_STACK_INIT_SIZE 256 +#define EXTERN_STACK_MAX_SIZE (1024 * 1024 * 100) + +struct extern_item { + uint64_t field_addr; + mlsize_t count; +}; + +/* Hash table to record already-marshaled objects and their positions */ + +struct object_position { + value obj; + uintnat pos; +}; + +/* The hash table uses open addressing, linear probing, and a redundant + representation: + - a bitvector [present] records which entries of the table are occupied; + - an array [entries] records (object, position) pairs for the entries + that are occupied. + The bitvector is much smaller than the array (1/128th on 64-bit + platforms), so it has better locality, making it faster to determine + that an object is not in the table. Also, it makes it faster to empty + or initialize a table: only the [present] bitvector needs to be filled + with zeros, the [entries] array can be left uninitialized. */ + +struct position_table { + int shift; + mlsize_t size; /* size == 1 << (wordsize - shift) */ + mlsize_t mask; /* mask == size - 1 */ + mlsize_t threshold; /* threshold == a fixed fraction of size */ + uintnat *present; /* [Bitvect_size(size)] */ + struct object_position *entries; /* [size] */ +}; + +#define Bits_word (8 * sizeof(uintnat)) +#define Bitvect_size(n) (((n) + Bits_word - 1) / Bits_word) + +#define POS_TABLE_INIT_SIZE_LOG2 8 +#define POS_TABLE_INIT_SIZE (1 << POS_TABLE_INIT_SIZE_LOG2) + +struct extern_state { + + /* How to read the debuggee's memory */ + MarshalMemoryReader *reader; + + uintnat obj_counter; /* Number of objects emitted so far */ + uintnat size_64; /* Size in words of 64-bit block for struct. */ + + /* Return point for error reporting */ + jmp_buf error_return; + extern_error_code_t error_code; + const char *error_msg; /* either a literal or error_buf */ + char error_buf[160]; + + /* Output size cap: bytes written into closed output blocks so far, and + the limit above which marshalling is abandoned */ + uintnat output_written; + uintnat max_output; + + /* Stack for pending values to marshal */ + struct extern_item extern_stack_init[EXTERN_STACK_INIT_SIZE]; + struct extern_item *extern_stack; + struct extern_item *extern_stack_limit; + + /* Hash table to record already marshalled objects */ + uintnat pos_table_present_init[Bitvect_size(POS_TABLE_INIT_SIZE)]; + struct object_position pos_table_entries_init[POS_TABLE_INIT_SIZE]; + struct position_table pos_table; + + /* To buffer the output */ + + char *extern_ptr; + char *extern_limit; + + struct output_block *extern_output_first; + struct output_block *extern_output_block; +}; + +[[noreturn]] static void extern_raise(struct extern_state *s, + extern_error_code_t code, + const char *msg) { + s->error_code = code; + s->error_msg = msg; + longjmp(s->error_return, 1); +} + +static void init_extern_stack(struct extern_state *s) { + /* (Re)initialize the globals for next time around */ + s->extern_stack = s->extern_stack_init; + s->extern_stack_limit = s->extern_stack + EXTERN_STACK_INIT_SIZE; +} + +/* Forward declarations */ + +[[noreturn]] static void extern_out_of_memory(struct extern_state *s); + +[[noreturn]] static void extern_invalid_argument(struct extern_state *s, + const char *msg); + +[[noreturn]] static void extern_stack_overflow(struct extern_state *s); + +[[noreturn]] static void extern_read_failure(struct extern_state *s, + uint64_t addr, uint64_t len); + +static void free_extern_output(struct extern_state *s); + +/* Reading the debuggee's memory. Both helpers raise (after freeing the + output) if the read fails, so callers can assume success. */ + +static void read_bytes(struct extern_state *s, uint64_t addr, void *buf, + uint64_t len) { + if (!s->reader->ReadBytes(addr, buf, len)) + extern_read_failure(s, addr, len); +} + +static uint64_t read_word(struct extern_state *s, uint64_t addr) { + uint64_t word; + if (!s->reader->ReadWord(addr, word)) + extern_read_failure(s, addr, sizeof(word)); + return word; +} + +/* The header word lives one word before the address a block pointer + designates (Hd_val in the runtime). */ +static header_t read_header(struct extern_state *s, value v) { + return (header_t)read_word(s, (uint64_t)v - sizeof(value)); +} + +/* Field(v, i) in the runtime */ +static value read_field(struct extern_state *s, value v, mlsize_t i) { + return (value)read_word(s, (uint64_t)v + i * sizeof(value)); +} + +/* caml_string_length in the runtime: the length is the block size in bytes + minus one, minus the padding count stored in the last byte. [wosize] is + the block size from the header (nonzero: atoms are handled earlier). */ +static mlsize_t string_length(struct extern_state *s, value v, + mlsize_t wosize) { + mlsize_t temp = wosize * sizeof(value) - 1; + uint8_t padding; + read_bytes(s, (uint64_t)v + temp, &padding, 1); + if (padding > sizeof(value) - 1) + extern_invalid_argument(s, "output_value: corrupted string block"); + return temp - padding; +} + +static void extern_free_stack(struct extern_state *s) { + /* Free the extern stack if needed */ + if (s->extern_stack != s->extern_stack_init) { + free(s->extern_stack); + init_extern_stack(s); + } +} + +static struct extern_item *extern_resize_stack(struct extern_state *s, + const struct extern_item *sp) { + asize_t newsize = 2 * (s->extern_stack_limit - s->extern_stack); + asize_t sp_offset = sp - s->extern_stack; + struct extern_item *newstack; + + if (newsize >= EXTERN_STACK_MAX_SIZE) + extern_stack_overflow(s); + newstack = + (struct extern_item *)calloc(newsize, sizeof(struct extern_item)); + if (newstack == NULL) + extern_stack_overflow(s); + + /* Copy items from the old stack to the new stack */ + memcpy(newstack, s->extern_stack, sizeof(struct extern_item) * sp_offset); + + /* Free the old stack if it is not the initial stack */ + if (s->extern_stack != s->extern_stack_init) + free(s->extern_stack); + + s->extern_stack = newstack; + s->extern_stack_limit = newstack + newsize; + return newstack + sp_offset; +} + +/* Multiplicative Fibonacci hashing + (Knuth, TAOCP vol 3, section 6.4, page 518). + HASH_FACTOR is (sqrt(5) - 1) / 2 * 2^wordsize. */ +#define HASH_FACTOR 11400714819323198486UL +#define Hash(v, shift) (((uintnat)(v) * HASH_FACTOR) >> (shift)) + +/* When the table becomes 2/3 full, its size is increased. */ +#define Threshold(sz) (((sz) * 2) / 3) + +/* Initialize the position table */ + +static void extern_init_position_table(struct extern_state *s) { + s->pos_table.size = POS_TABLE_INIT_SIZE; + s->pos_table.shift = 8 * sizeof(value) - POS_TABLE_INIT_SIZE_LOG2; + s->pos_table.mask = POS_TABLE_INIT_SIZE - 1; + s->pos_table.threshold = Threshold(POS_TABLE_INIT_SIZE); + s->pos_table.present = s->pos_table_present_init; + s->pos_table.entries = s->pos_table_entries_init; + memset(s->pos_table_present_init, 0, + Bitvect_size(POS_TABLE_INIT_SIZE) * sizeof(uintnat)); +} + +/* Free the position table */ + +static void extern_free_position_table(struct extern_state *s) { + if (s->pos_table.present != s->pos_table_present_init) { + free(s->pos_table.present); + free(s->pos_table.entries); + /* Protect against repeated calls to extern_free_position_table */ + s->pos_table.present = s->pos_table_present_init; + s->pos_table.entries = s->pos_table_entries_init; + } +} + +/* Accessing bitvectors */ + +static inline uintnat bitvect_test(uintnat *bv, uintnat i) { + return bv[i / Bits_word] & ((uintnat)1 << (i & (Bits_word - 1))); +} + +static inline void bitvect_set(uintnat *bv, uintnat i) { + bv[i / Bits_word] |= ((uintnat)1 << (i & (Bits_word - 1))); +} + +/* Grow the position table */ + +static void extern_resize_position_table(struct extern_state *s) { + mlsize_t new_size, new_byte_size; + int new_shift; + uintnat *new_present; + struct object_position *new_entries; + uintnat h; + struct position_table old = s->pos_table; + + /* Grow the table quickly (x 8) up to 10^6 entries, + more slowly (x 2) afterwards. */ + if (old.size < 1000000) { + new_size = 8 * old.size; + new_shift = old.shift - 3; + } else { + new_size = 2 * old.size; + new_shift = old.shift - 1; + } + if (new_size == 0 || + umul_overflow(new_size, sizeof(struct object_position), &new_byte_size)) + extern_out_of_memory(s); + new_entries = (struct object_position *)malloc(new_byte_size); + if (new_entries == NULL) + extern_out_of_memory(s); + new_present = (uintnat *)calloc(Bitvect_size(new_size), sizeof(uintnat)); + if (new_present == NULL) { + free(new_entries); + extern_out_of_memory(s); + } + s->pos_table.size = new_size; + s->pos_table.shift = new_shift; + s->pos_table.mask = new_size - 1; + s->pos_table.threshold = Threshold(new_size); + s->pos_table.present = new_present; + s->pos_table.entries = new_entries; + + /* Insert every entry of the old table in the new table */ + for (uintnat i = 0; i < old.size; i++) { + if (!bitvect_test(old.present, i)) + continue; + h = Hash(old.entries[i].obj, s->pos_table.shift); + while (bitvect_test(new_present, h)) { + h = (h + 1) & s->pos_table.mask; + } + bitvect_set(new_present, h); + new_entries[h] = old.entries[i]; + } + + /* Free the old tables if they are not the initial ones */ + if (old.present != s->pos_table_present_init) { + free(old.present); + free(old.entries); + } +} + +/* Determine whether the given object [obj] is in the hash table. + If so, set [*pos_out] to its position in the output and return 1. + If not, return 0. + Either way, set [*h_out] to the hash value appropriate for + [extern_record_location]. */ +static inline int extern_lookup_position(struct extern_state *s, value obj, + uintnat *pos_out, uintnat *h_out) { + uintnat h = Hash(obj, s->pos_table.shift); + while (1) { + if (!bitvect_test(s->pos_table.present, h)) { + *h_out = h; + return 0; + } + if (s->pos_table.entries[h].obj == obj) { + *h_out = h; + *pos_out = s->pos_table.entries[h].pos; + return 1; + } + h = (h + 1) & s->pos_table.mask; + } +} + +/* Record the output position for the given object [obj]. */ +/* The [h] parameter is the index in the hash table where the object + must be inserted. It was determined during lookup. */ +static void extern_record_location(struct extern_state *s, value obj, + uintnat h) { + bitvect_set(s->pos_table.present, h); + s->pos_table.entries[h].obj = obj; + s->pos_table.entries[h].pos = s->obj_counter; + s->obj_counter++; + if (s->obj_counter >= s->pos_table.threshold) + extern_resize_position_table(s); +} + +/* To buffer the output */ + +static void init_extern_output(struct extern_state *s) { + s->extern_output_first = + (struct output_block *)malloc(sizeof(struct output_block)); + if (s->extern_output_first == NULL) + extern_raise(s, EXTERN_ERROR_OUT_OF_MEMORY, + "output_value: out of memory"); + s->extern_output_block = s->extern_output_first; + s->extern_output_block->next = NULL; + s->extern_ptr = s->extern_output_block->data; + s->extern_limit = s->extern_output_block->data + SIZE_EXTERN_OUTPUT_BLOCK; + s->output_written = 0; +} + +static void close_extern_output(struct extern_state *s) { + s->extern_output_block->end = s->extern_ptr; +} + +static void free_extern_output(struct extern_state *s) { + for (struct output_block *blk = s->extern_output_first, *nextblk; + blk != NULL; blk = nextblk) { + nextblk = blk->next; + free(blk); + } + s->extern_output_first = NULL; + extern_free_stack(s); + extern_free_position_table(s); +} + +[[noreturn]] static void extern_output_too_large(struct extern_state *s) { + free_extern_output(s); + extern_raise(s, EXTERN_ERROR_OUTPUT_TOO_LARGE, + "output_value: marshalled representation exceeds the " + "configured size limit"); +} + +static void grow_extern_output(struct extern_state *s, intnat required) { + struct output_block *blk; + intnat extra; + + s->extern_output_block->end = s->extern_ptr; + /* Enforce the output cap here rather than on every byte written: a new + block is requested at worst every SIZE_EXTERN_OUTPUT_BLOCK bytes, so + the cap can only be overshot by one block. */ + s->output_written += s->extern_ptr - s->extern_output_block->data; + if (s->output_written + (uintnat)required > s->max_output) + extern_output_too_large(s); + if (required <= SIZE_EXTERN_OUTPUT_BLOCK / 2) + extra = 0; + else + extra = required; + blk = (struct output_block *)malloc(sizeof(struct output_block) + extra); + if (blk == NULL) + extern_out_of_memory(s); + s->extern_output_block->next = blk; + s->extern_output_block = blk; + s->extern_output_block->next = NULL; + s->extern_ptr = s->extern_output_block->data; + s->extern_limit = + s->extern_output_block->data + SIZE_EXTERN_OUTPUT_BLOCK + extra; +} + +static intnat extern_output_length(struct extern_state *s) { + struct output_block *blk; + intnat len; + + for (len = 0, blk = s->extern_output_first; blk != NULL; blk = blk->next) + len += blk->end - blk->data; + return len; +} + +/* Error raising, with cleanup */ + +static void extern_out_of_memory(struct extern_state *s) { + free_extern_output(s); + extern_raise(s, EXTERN_ERROR_OUT_OF_MEMORY, "output_value: out of memory"); +} + +static void extern_invalid_argument(struct extern_state *s, const char *msg) { + free_extern_output(s); + extern_raise(s, EXTERN_ERROR_INVALID_ARGUMENT, msg); +} + +static void extern_stack_overflow(struct extern_state *s) { + free_extern_output(s); + extern_raise(s, EXTERN_ERROR_OUT_OF_MEMORY, + "output_value: stack overflow"); +} + +static void extern_read_failure(struct extern_state *s, uint64_t addr, + uint64_t len) { + free_extern_output(s); + snprintf(s->error_buf, sizeof(s->error_buf), + "output_value: cannot read %llu byte(s) at address 0x%llx in the " + "debugged process", + (unsigned long long)len, (unsigned long long)addr); + extern_raise(s, EXTERN_ERROR_READ_FAILURE, s->error_buf); +} + +/* Conversion to big-endian */ + +static inline void store16(char *dst, int n) { + uint16_t u = Htobe16(n); + memcpy(dst, &u, 2); +} + +static inline void store32(char *dst, intnat n) { + uint32_t u = Htobe32(n); + memcpy(dst, &u, 4); +} + +static inline void store64(char *dst, int64_t n) { + uint64_t u = Htobe64(n); + memcpy(dst, &u, 8); +} + +/* Write characters, integers, and blocks in the output buffer */ + +static inline void writebyte(struct extern_state *s, int c) { + if (s->extern_ptr >= s->extern_limit) + grow_extern_output(s, 1); + *s->extern_ptr++ = c; +} + +static void writeblock(struct extern_state *s, const char *data, intnat len) { + if (s->extern_ptr + len > s->extern_limit) + grow_extern_output(s, len); + memcpy(s->extern_ptr, data, len); + s->extern_ptr += len; +} + +/* Copy [len] bytes from debuggee address [addr] into the output. The copy + goes through a bounded host buffer so that a huge (possibly corrupt) + length does not translate into a huge host allocation before the output + cap check in grow_extern_output can fire. */ +static void writeblock_from_target(struct extern_state *s, uint64_t addr, + intnat len) { + char chunk[4096]; + while (len > 0) { + intnat n = len < (intnat)sizeof(chunk) ? len : (intnat)sizeof(chunk); + read_bytes(s, addr, chunk, n); + writeblock(s, chunk, n); + addr += n; + len -= n; + } +} + +static void writecode8(struct extern_state *s, int code, intnat val) { + if (s->extern_ptr + 2 > s->extern_limit) + grow_extern_output(s, 2); + s->extern_ptr[0] = code; + s->extern_ptr[1] = val; + s->extern_ptr += 2; +} + +static void writecode16(struct extern_state *s, int code, intnat val) { + if (s->extern_ptr + 3 > s->extern_limit) + grow_extern_output(s, 3); + s->extern_ptr[0] = code; + store16(s->extern_ptr + 1, (int)val); + s->extern_ptr += 3; +} + +static void writecode32(struct extern_state *s, int code, intnat val) { + if (s->extern_ptr + 5 > s->extern_limit) + grow_extern_output(s, 5); + s->extern_ptr[0] = code; + store32(s->extern_ptr + 1, val); + s->extern_ptr += 5; +} + +static void writecode64(struct extern_state *s, int code, intnat val) { + if (s->extern_ptr + 9 > s->extern_limit) + grow_extern_output(s, 9); + s->extern_ptr[0] = code; + store64(s->extern_ptr + 1, val); + s->extern_ptr += 9; +} + +/* Marshaling integers */ + +static inline void extern_int(struct extern_state *s, intnat n) { + if (n >= 0 && n < 0x40) { + writebyte(s, PREFIX_SMALL_INT + n); + } else if (n >= -(1 << 7) && n < (1 << 7)) { + writecode8(s, CODE_INT8, n); + } else if (n >= -(1 << 15) && n < (1 << 15)) { + writecode16(s, CODE_INT16, n); + } else if (n < -((intnat)1 << 30) || n >= ((intnat)1 << 30)) { + writecode64(s, CODE_INT64, n); + } else { + writecode32(s, CODE_INT32, n); + } +} + +static inline void extern_null(struct extern_state *s) { + writecode8(s, CODE_NULL, 0); +} + +/* Marshaling references to previously-marshaled blocks */ + +static inline void extern_shared_reference(struct extern_state *s, uintnat d) { + if (d < 0x100) { + writecode8(s, CODE_SHARED8, d); + } else if (d < 0x10000) { + writecode16(s, CODE_SHARED16, d); + } else if (d >= (uintnat)1 << 32) { + writecode64(s, CODE_SHARED64, d); + } else { + writecode32(s, CODE_SHARED32, d); + } +} + +/* Marshaling block headers */ + +static inline void extern_header(struct extern_state *s, mlsize_t sz, + tag_t tag) { + if (tag < 16 && sz < 8) { + writebyte(s, PREFIX_SMALL_BLOCK + tag + (sz << 4)); + } else { + header_t hd = Make_header(sz, tag); + if (hd < (uintnat)1 << 32) + writecode32(s, CODE_BLOCK32, hd); + else + writecode64(s, CODE_BLOCK64, hd); + } +} + +/* Marshaling strings */ + +static inline void extern_string(struct extern_state *s, value v, + mlsize_t len) { + if (len < 0x20) { + writebyte(s, PREFIX_SMALL_STRING + len); + } else if (len < 0x100) { + writecode8(s, CODE_STRING8, len); + } else { + if (len < (uintnat)1 << 32) + writecode32(s, CODE_STRING32, len); + else + writecode64(s, CODE_STRING64, len); + } + writeblock_from_target(s, (uint64_t)v, len); +} + +/* Marshaling FP numbers. Doubles are assumed to be IEEE 754 with the same + byte order as integers on both the host and the debuggee; the + CODE_DOUBLE*_NATIVE codes record which endianness that is. */ + +static inline void extern_double(struct extern_state *s, value v) { + writebyte(s, CODE_DOUBLE_NATIVE); + writeblock_from_target(s, (uint64_t)v, 8); +} + +/* Marshaling FP arrays */ + +static inline void extern_double_array(struct extern_state *s, value v, + mlsize_t nfloats) { + if (nfloats < 0x100) { + writecode8(s, CODE_DOUBLE_ARRAY8_NATIVE, nfloats); + } else { + if (nfloats < (uintnat)1 << 32) + writecode32(s, CODE_DOUBLE_ARRAY32_NATIVE, nfloats); + else + writecode64(s, CODE_DOUBLE_ARRAY64_NATIVE, nfloats); + } + writeblock_from_target(s, (uint64_t)v, nfloats * 8); +} + +/* Marshal the given value in the output buffer */ + +static void extern_rec(struct extern_state *s, value v) { + struct extern_item *sp; + uintnat h = 0; + uintnat pos = 0; + + /* for Double_tag and Double_array_tag */ + static_assert(sizeof(double) == 8, ""); + + extern_init_position_table(s); + sp = s->extern_stack; + + while (1) { + if (Is_null(v)) { + extern_null(s); + } else if (Is_long(v)) { + extern_int(s, Long_val(v)); + } else { + header_t hd = read_header(s, v); + tag_t tag = Tag_hd(hd); + mlsize_t sz = Wosize_hd(hd); + if (Reserved_hd(hd) != 0) { + /* Nonzero reserved header bits indicate a mixed block. */ + extern_invalid_argument(s, "output_value: mixed block"); + break; + } + + if (tag == Forward_tag) { + value f = read_field(s, v, 0); + tag_t ftag = Is_block(f) ? Tag_hd(read_header(s, f)) : 0; + if (Is_block(f) && (ftag == Forward_tag || ftag == Lazy_tag || + ftag == Forcing_tag || + /* Double_tag check because of flat float + arrays */ + ftag == Double_tag)) { + /* Do not short-circuit the pointer. */ + } else { + v = f; + continue; + } + } + /* Atoms are treated specially for two reasons: they are not allocated + in the externed block, and they are automatically shared. */ + if (sz == 0) { + extern_header(s, 0, tag); + goto next_item; + } + /* Check if object already seen */ + if (extern_lookup_position(s, v, &pos, &h)) { + extern_shared_reference(s, s->obj_counter - pos); + goto next_item; + } + /* Output the contents of the object */ + switch (tag) { + case String_tag: { + mlsize_t len = string_length(s, v, sz); + extern_string(s, v, len); + s->size_64 += 1 + (len + 8) / 8; + extern_record_location(s, v, h); + break; + } + case Double_tag: { + extern_double(s, v); + s->size_64 += 1 + 1; + extern_record_location(s, v, h); + break; + } + case Double_array_tag: { + /* sizeof(double) == sizeof(value), per the static_assert above */ + mlsize_t nfloats = sz; + extern_double_array(s, v, nfloats); + s->size_64 += 1 + nfloats; + extern_record_location(s, v, h); + break; + } + case Abstract_tag: + extern_invalid_argument(s, + "output_value: abstract value (Abstract)"); + break; + case Infix_tag: + /* An infix pointer into a closure block; closures cannot be + marshalled (see Closure_tag below). */ + extern_invalid_argument(s, "output_value: functional value"); + break; + case Custom_tag: + /* Custom blocks (Int32, Int64, Nativeint, Bigarray, ...) cannot + be marshalled here: their serialization functions live in the + debuggee and cannot be run. */ + extern_invalid_argument(s, "output_value: abstract value (Custom)"); + break; + case Closure_tag: + /* The runtime's code-fragment table is not available here, so + code pointers, and hence closures, cannot be marshalled. */ + extern_invalid_argument(s, "output_value: functional value"); + break; + case Cont_tag: + extern_invalid_argument(s, "output_value: continuation value"); + break; + default: { + extern_header(s, sz, tag); + s->size_64 += 1 + sz; + extern_record_location(s, v, h); + /* Remember that we still have to serialize fields 1 ... sz - 1 */ + if (sz > 1) { + sp++; + if (sp >= s->extern_stack_limit) + sp = extern_resize_stack(s, sp); + sp->field_addr = (uint64_t)v + sizeof(value); + sp->count = sz - 1; + } + /* Continue serialization with the first field */ + v = read_field(s, v, 0); + continue; + } + } + } + next_item: + /* Pop one more item to marshal, if any */ + if (sp == s->extern_stack) { + /* We are done. Cleanup the stack and leave the function */ + extern_free_stack(s); + extern_free_position_table(s); + return; + } + v = (value)read_word(s, sp->field_addr); + sp->field_addr += sizeof(value); + if (--(sp->count) == 0) + sp--; + } + /* Never reached as function leaves with return */ +} + +static intnat extern_value(struct extern_state *s, value v, + /*out*/ char header[MAX_INTEXT_HEADER_SIZE], + /*out*/ int *header_len) { + intnat res_len; + /* Initializations */ + s->obj_counter = 0; + s->size_64 = 0; + /* Marshal the object */ + extern_rec(s, v); + /* Record end of output */ + close_extern_output(s); + /* Write the header */ + res_len = extern_output_length(s); + if (res_len >= ((intnat)1 << 32) || s->size_64 >= ((uintnat)1 << 32)) { + /* The object is too big for the small header format. + Use the big header. */ + store32(header, Intext_magic_number_big); + store32(header + 4, 0); + store64(header + 8, res_len); + store64(header + 16, s->obj_counter); + store64(header + 24, s->size_64); + *header_len = 32; + return res_len; + } + /* Use the small header format. The field at offset 12 is the size in + words when read on a 32-bit platform; 32-bit readers are not + supported, so 0 is written instead. */ + store32(header, Intext_magic_number_small); + store32(header + 4, res_len); + store32(header + 8, s->obj_counter); + store32(header + 12, 0); + store32(header + 16, s->size_64); + *header_len = 20; + return res_len; +} + +/* The entry point. On success returns EXTERN_OK and stores a malloc'd + buffer and its length in [*buf] and [*len]; the caller frees the buffer. + On failure returns the error code (with the message in s->error_msg) and + writes neither output. */ + +static extern_error_code_t output_value_to_malloc(struct extern_state *s, + value v, + /*out*/ char **buf, + /*out*/ intnat *len) { + char header[MAX_INTEXT_HEADER_SIZE]; + int header_len; + intnat data_len; + char *res; + + s->error_code = EXTERN_OK; + s->error_msg = NULL; + if (setjmp(s->error_return) != 0) + return s->error_code; + + init_extern_output(s); + data_len = extern_value(s, v, header, &header_len); + res = (char *)malloc(header_len + data_len); + if (res == NULL) + extern_out_of_memory(s); + *buf = res; + *len = header_len + data_len; + memcpy(res, header, header_len); + res += header_len; + for (struct output_block *blk = s->extern_output_first, *nextblk; + blk != NULL; blk = nextblk) { + intnat n = blk->end - blk->data; + memcpy(res, blk->data, n); + res += n; + nextblk = blk->next; + free(blk); + } + s->extern_output_first = NULL; + return EXTERN_OK; +} + +} // namespace + +// --------------------------------------------------------------------------- +// Public entry point + +llvm::Expected> +lldb_private::formatters::oxcaml::MarshalOCamlValue(uint64_t root, + MarshalMemoryReader &reader, + uint64_t max_output_bytes) { + // The state is large (~16 KiB of inline stacks and tables), so it lives on + // the heap. It is plain data: no constructors or destructors may exist + // inside the setjmp region. + struct extern_state *s = + (struct extern_state *)calloc(1, sizeof(struct extern_state)); + if (s == NULL) + return llvm::createStringError( + "output_value: out of memory allocating marshalling state"); + s->reader = &reader; + s->max_output = max_output_bytes; + init_extern_stack(s); + + char *buf = NULL; + intnat len = 0; + extern_error_code_t err = + output_value_to_malloc(s, (value)root, &buf, &len); + if (err != EXTERN_OK) { + std::string msg(s->error_msg ? s->error_msg + : "output_value: unknown error"); + free(s); + return llvm::createStringError("%s", msg.c_str()); + } + free(s); + + std::vector result(buf, buf + len); + free(buf); + return result; +} diff --git a/lldb/source/Plugins/Language/OxCaml/OxCamlMarshal.h b/lldb/source/Plugins/Language/OxCaml/OxCamlMarshal.h new file mode 100644 index 0000000000000..006690a43ea9e --- /dev/null +++ b/lldb/source/Plugins/Language/OxCaml/OxCamlMarshal.h @@ -0,0 +1,83 @@ +//===-- OxCamlMarshal.h -----------------------------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// A marshaller ("extern"), producing OCaml's Marshal wire format, that walks +// an OCaml object graph living in a debugged process rather than in the +// marshaller's own address space. Adapted from the OxCaml runtime's +// runtime/extern.c (via extern_standalone.cpp in this repository); all reads +// of the debuggee's heap go through a MarshalMemoryReader instead of raw +// pointer dereferences. +// +// The output is accepted by Marshal.from_channel / Marshal.from_bytes in an +// OxCaml program, which is how the external pretty-printer scheme consumes +// it (see OxCamlExternalPrinter.cpp). +// +//===----------------------------------------------------------------------===// + +#ifndef LLDB_SOURCE_PLUGINS_LANGUAGE_OXCAML_OXCAMLMARSHAL_H +#define LLDB_SOURCE_PLUGINS_LANGUAGE_OXCAML_OXCAMLMARSHAL_H + +#include "lldb/lldb-forward.h" +#include "llvm/Support/Error.h" +#include +#include + +namespace lldb_private { +namespace formatters { +namespace oxcaml { + +/// Access to the debuggee's memory for the marshaller. Reads go through +/// LLDB rather than raw pointers so that the object graph can live in a +/// debugged process; other implementations (for example one materializing +/// values described by DWARF expressions, such as implicit pointers) can be +/// substituted without touching the marshaller itself. +class MarshalMemoryReader { +public: + virtual ~MarshalMemoryReader() = default; + + /// Read \p len bytes at target address \p addr into \p buf. Returns + /// false if any part of the range cannot be read. + virtual bool ReadBytes(uint64_t addr, void *buf, uint64_t len) = 0; + + /// Read one 8-byte word at target address \p addr. The default + /// implementation reads the bytes and assumes the debuggee shares the + /// host's byte order (true for all supported OxCaml targets). + virtual bool ReadWord(uint64_t addr, uint64_t &word); +}; + +/// A MarshalMemoryReader that reads from a live process. +class ProcessMemoryReader : public MarshalMemoryReader { +public: + explicit ProcessMemoryReader(lldb::ProcessSP process_sp) + : m_process_sp(std::move(process_sp)) {} + + bool ReadBytes(uint64_t addr, void *buf, uint64_t len) override; + +private: + lldb::ProcessSP m_process_sp; +}; + +/// Marshal the OCaml value \p root (a tagged word: an immediate, the null +/// value, or a pointer to a heap block) into OCaml's Marshal wire format, +/// following heap pointers through \p reader. On success the returned +/// buffer holds the complete marshalled representation, header included, +/// as expected by Marshal.from_channel. Sharing within the value is +/// preserved. +/// +/// Values with no external representation (closures, continuations, +/// custom / abstract / mixed blocks), unreadable memory, and outputs that +/// would exceed \p max_output_bytes all yield an error. +llvm::Expected> +MarshalOCamlValue(uint64_t root, MarshalMemoryReader &reader, + uint64_t max_output_bytes = 64 * 1024 * 1024); + +} // namespace oxcaml +} // namespace formatters +} // namespace lldb_private + +#endif // LLDB_SOURCE_PLUGINS_LANGUAGE_OXCAML_OXCAMLMARSHAL_H diff --git a/lldb/source/Plugins/SymbolFile/DWARF/DWARFASTParserOxCaml.cpp b/lldb/source/Plugins/SymbolFile/DWARF/DWARFASTParserOxCaml.cpp index 60a63ebc3e2c5..c8bfcce290ced 100644 --- a/lldb/source/Plugins/SymbolFile/DWARF/DWARFASTParserOxCaml.cpp +++ b/lldb/source/Plugins/SymbolFile/DWARF/DWARFASTParserOxCaml.cpp @@ -8,16 +8,22 @@ #include "DWARFASTParserOxCaml.h" +#include "DWARFAttribute.h" #include "DWARFDIE.h" #include "DWARFDebugInfoEntry.h" #include "DWARFDefines.h" +#include "DWARFFormValue.h" +#include "DWARFUnit.h" #include "SymbolFileDWARF.h" #include "llvm/BinaryFormat/Dwarf.h" +#include "lldb/Expression/DWARFExpression.h" +#include "lldb/Expression/DWARFExpressionList.h" #include "lldb/Symbol/Type.h" #include "../../Language/OxCaml/LogChannelOxCaml.h" #include "../../Language/OxCaml/OxCamlAssert.h" +#include "Plugins/TypeSystem/OxCaml/OxCamlDynamicLayoutValue.h" #include "Plugins/TypeSystem/OxCaml/TypeSystemOxCaml.h" #include "lldb/Core/Module.h" #include "lldb/Symbol/CompileUnit.h" @@ -29,18 +35,56 @@ using namespace lldb; using namespace lldb_private; using namespace lldb_private::plugin::dwarf; -// Custom OCaml DWARF attribute DW_AT_ocaml_offset_record_from_pointer -// This attribute indicates the offset to apply when dereferencing pointers to -// this type -static constexpr uint32_t DW_AT_ocaml_offset_record_from_pointer = 0x3106; -// CR sspies: Technically, this requires an extension of the DWARF -// standard. If we want to upstream this, we should think about whether the -// attribute should be on the structure or on the pointer itself (indicating a -// base offset). For now, we declare it as an ad-hoc attribute here. - // Default size for OCaml values when size attribute is missing static constexpr uint64_t DEFAULT_OCAML_VALUE_SIZE = 8; +namespace { + +// Locate a single attribute on a DIE and return its form value. Returns +// std::nullopt if the attribute is absent or the form value cannot be +// extracted. +std::optional +GetFormValueForAttribute(const DWARFDIE &die, llvm::dwarf::Attribute attr) { + DWARFAttributes attrs = die.GetAttributes(); + for (uint32_t i = 0; i < attrs.Size(); ++i) { + if (attrs.AttributeAtIndex(i) != attr) + continue; + DWARFFormValue form_value; + if (!attrs.ExtractFormValueAtIndex(i, form_value)) + return std::nullopt; + return form_value; + } + return std::nullopt; +} + +// Parse a layout attribute that may be a constant or a DW_FORM_exprloc. +// The (kind, initial_stack) pair is fixed per attribute by the compiler +// contract; callers pass the pair that matches the attribute. +std::optional ParseConstantOrExpression( + const DWARFDIE &die, llvm::dwarf::Attribute attr, + OxCamlLayoutValueKind kind, + OxCamlDynamicLayoutValue::InitialStackLayout initial_stack) { + auto form_value = GetFormValueForAttribute(die, attr); + if (!form_value.has_value()) + return std::nullopt; + + if (form_value->BlockData() == nullptr) + return OxCamlDynamicLayoutValue::MakeConstant(form_value->Unsigned(), kind); + + const DWARFDataExtractor &debug_info_data = die.GetData(); + uint32_t block_length = form_value->Unsigned(); + uint32_t block_offset = + form_value->BlockData() - debug_info_data.GetDataStart(); + DataExtractor expr_data(debug_info_data, block_offset, block_length); + DWARFExpression expr(expr_data); + + DWARFExpressionList list(die.GetModule(), std::move(expr), die.GetCU()); + return OxCamlDynamicLayoutValue::MakeExpression(std::move(list), kind, + initial_stack); +} + +} // namespace + DWARFASTParserOxCaml::DWARFASTParserOxCaml(TypeSystemOxCaml &oxcaml_typesystem) : DWARFASTParser(Kind::DWARFASTParserOxCaml), m_oxcaml_typesystem(oxcaml_typesystem) {} @@ -512,32 +556,38 @@ DWARFASTParserOxCaml::ParseStructureType(const SymbolContext &sc, LLDB_LOG(log, "ParseStructureType: DIE 0x{0:x16}", die_id); - std::optional byte_size_opt = - die.GetAttributeValueAsOptionalUnsigned(llvm::dwarf::DW_AT_byte_size); - if (!byte_size_opt.has_value() || byte_size_opt.value() == 0) { + auto size_opt = ParseConstantOrExpression( + die, llvm::dwarf::DW_AT_byte_size, OxCamlLayoutValueKind::ScalarBytes, + OxCamlDynamicLayoutValue::InitialStackLayout::Empty); + if (!size_opt.has_value()) { + size_opt = ParseConstantOrExpression( + die, llvm::dwarf::DW_AT_bit_size, OxCamlLayoutValueKind::ScalarBits, + OxCamlDynamicLayoutValue::InitialStackLayout::Empty); + } + if (!size_opt.has_value()) { LLDB_LOG(log, - "ParseStructureType: Missing or zero byte_size for DIE 0x{0:x16}", + "ParseStructureType: Missing byte_size and bit_size for DIE " + "0x{0:x16}", die_id); return nullptr; } - uint64_t byte_size = byte_size_opt.value(); + OxCamlDynamicLayoutValue size = std::move(*size_opt); - std::optional name = ExtractTypeName(die); - - auto ocaml_attr = static_cast( - DW_AT_ocaml_offset_record_from_pointer); - std::optional attr_value_opt = - die.GetAttributeValueAsOptionalUnsigned(ocaml_attr); - - int64_t base_offset = 0; - if (attr_value_opt.has_value()) { - base_offset = static_cast(attr_value_opt.value()); - LLDB_LOG( - log, - "ParseStructureType: Found DW_AT_ocaml_offset_record_from_pointer: {0}", - base_offset); + uint64_t static_size_fallback; + if (size.IsConstant()) { + if (size.GetConstant() == 0) { + LLDB_LOG(log, "ParseStructureType: Zero size for DIE 0x{0:x16}", die_id); + return nullptr; + } + static_size_fallback = size.GetKind() == OxCamlLayoutValueKind::ScalarBits + ? (size.GetConstant() + 7) / 8 + : size.GetConstant(); + } else { + static_size_fallback = formatters::oxcaml::helpers::constants::WORD_SIZE; } + std::optional name = ExtractTypeName(die); + std::vector members; std::vector variant_parts; @@ -546,11 +596,14 @@ DWARFASTParserOxCaml::ParseStructureType(const SymbolContext &sc, case llvm::dwarf::DW_TAG_member: { auto member = ParseMember(sc, child_die); if (member.has_value()) { - members.push_back(std::move(*member)); + const char *location_kind = + member->location.IsConstant() ? "constant" : "expression"; LLDB_LOG( - log, "ParseStructureType: Added member {0} at offset {1}, type {2}", - member->name.value_or(""), member->data_member_location, + log, + "ParseStructureType: Added member {0} ({1} location), type {2}", + member->name.value_or(""), location_kind, member->GetType()->GetDisplayName()); + members.push_back(std::move(*member)); } else { LLDB_LOG(log, "ParseStructureType: Failed to parse member, skipping"); } @@ -576,12 +629,12 @@ DWARFASTParserOxCaml::ParseStructureType(const SymbolContext &sc, LLDB_LOG(log, "ParseStructureType: Creating OxCamlStructureType with {0} members, " - "{1} variant parts, size {2}", - members.size(), variant_parts.size(), byte_size); + "{1} variant parts, static-size-fallback {2}", + members.size(), variant_parts.size(), static_size_fallback); return std::make_unique( - die_id, std::move(name), byte_size, std::move(members), - std::move(variant_parts), base_offset); + die_id, std::move(name), std::move(size), static_size_fallback, + std::move(members), std::move(variant_parts)); } std::optional @@ -613,22 +666,24 @@ DWARFASTParserOxCaml::ParseMember(const SymbolContext &sc, member_type_die.GetID()); Reference *member_type_ref = member_type_opt.value(); - std::optional member_offset_opt = - member_die.GetAttributeValueAsOptionalUnsigned( - llvm::dwarf::DW_AT_data_member_location); - if (!member_offset_opt.has_value()) { + auto location_opt = ParseConstantOrExpression( + member_die, llvm::dwarf::DW_AT_data_member_location, + OxCamlLayoutValueKind::Location, + OxCamlDynamicLayoutValue::InitialStackLayout::ObjectAddress); + if (!location_opt.has_value()) { LLDB_LOG(log, "ParseMember: Missing data_member_location for DIE 0x{0:x16}", member_die.GetID()); return std::nullopt; } - uint64_t member_offset = member_offset_opt.value(); - std::optional bit_offset = - member_die.GetAttributeValueAsOptionalUnsigned( - llvm::dwarf::DW_AT_data_bit_offset); - std::optional bit_size = - member_die.GetAttributeValueAsOptionalUnsigned( - llvm::dwarf::DW_AT_bit_size); + auto bit_offset = ParseConstantOrExpression( + member_die, llvm::dwarf::DW_AT_data_bit_offset, + OxCamlLayoutValueKind::ScalarBits, + OxCamlDynamicLayoutValue::InitialStackLayout::Empty); + auto bit_size = ParseConstantOrExpression( + member_die, llvm::dwarf::DW_AT_bit_size, + OxCamlLayoutValueKind::ScalarBits, + OxCamlDynamicLayoutValue::InitialStackLayout::Empty); std::optional artificial_opt = member_die.GetAttributeValueAsOptionalUnsigned( @@ -636,22 +691,20 @@ DWARFASTParserOxCaml::ParseMember(const SymbolContext &sc, bool is_artificial = artificial_opt.has_value() && artificial_opt.value() != 0; - LLDB_LOG(log, "ParseMember: Created member {0} at offset {1}, type {2}{3}", - member_name.value_or(""), member_offset, + LLDB_LOG(log, "ParseMember: Created member {0} ({1}), type {2}{3}", + member_name.value_or(""), + location_opt->IsConstant() ? "constant location" : "expr location", member_type_ref->get()->GetDisplayName(), is_artificial ? " (artificial)" : ""); if (bit_offset.has_value() || bit_size.has_value()) { - LLDB_LOG(log, "ParseMember: Bit field detected - offset: {0}, size: {1}", - bit_offset.value_or(0), bit_size.value_or(0)); + LLDB_LOG(log, "ParseMember: Bit field detected for DIE 0x{0:x16}", + member_die.GetID()); } - return OxCamlMember{std::move(member_name), - member_type_ref, - member_offset, - bit_offset, - bit_size, - is_artificial}; + return OxCamlMember{std::move(member_name), member_type_ref, + std::move(*location_opt), std::move(bit_offset), + std::move(bit_size), is_artificial}; } std::optional @@ -678,8 +731,9 @@ DWARFASTParserOxCaml::ParseVariantPart(const SymbolContext &sc, return std::nullopt; } - LLDB_LOG(log, "ParseVariantPart: Discriminator at offset {0}, type: {1}", - discriminator_member->data_member_location, + LLDB_LOG(log, "ParseVariantPart: Discriminator ({0} location), type: {1}", + discriminator_member->location.IsConstant() ? "constant" + : "expression", discriminator_member->GetType()->GetDisplayName()); std::vector variants; @@ -755,40 +809,37 @@ DWARFASTParserOxCaml::ParseArrayType(const SymbolContext &sc, element_type_die.GetID()); Reference *element_type_ref = element_type_opt.value(); - std::optional byte_stride_opt = - die.GetAttributeValueAsOptionalUnsigned(llvm::dwarf::DW_AT_byte_stride); - uint64_t byte_stride; - if (byte_stride_opt.has_value()) { - byte_stride = byte_stride_opt.value(); - } else { - byte_stride = element_type_ref->get()->GetByteSize(); - LLDB_LOG(log, - "ParseArrayType: No explicit byte_stride, using element size: {0}", - byte_stride); - } + auto byte_stride_opt = ParseConstantOrExpression( + die, llvm::dwarf::DW_AT_byte_stride, OxCamlLayoutValueKind::ScalarBytes, + OxCamlDynamicLayoutValue::InitialStackLayout::Empty); + OxCamlDynamicLayoutValue byte_stride = + byte_stride_opt.has_value() ? std::move(*byte_stride_opt) + : OxCamlDynamicLayoutValue::MakeConstant( + element_type_ref->get()->GetByteSize(), + OxCamlLayoutValueKind::ScalarBytes); - std::optional count; + std::optional count; for (DWARFDIE child_die : die.children()) { if (child_die.Tag() == llvm::dwarf::DW_TAG_subrange_type) { - auto subrange_count_opt = child_die.GetAttributeValueAsOptionalUnsigned( - llvm::dwarf::DW_AT_count); - if (subrange_count_opt.has_value()) { - count = subrange_count_opt.value(); - LLDB_LOG(log, "ParseArrayType: Found subrange count: {0}", - count.value()); - } + count = ParseConstantOrExpression( + child_die, llvm::dwarf::DW_AT_count, + OxCamlLayoutValueKind::ScalarElements, + OxCamlDynamicLayoutValue::InitialStackLayout::Empty); break; } } LLDB_LOG(log, "ParseArrayType: Creating OxCamlArrayType with element type {0}, " - "stride {1}, count {2}", - element_type_ref->get()->GetDisplayName(), byte_stride, - count.has_value() ? std::to_string(count.value()) : "unknown"); - - return std::make_unique( - die_id, std::move(name), element_type_ref, count, byte_stride); + "{1} stride, {2} count", + element_type_ref->get()->GetDisplayName(), + byte_stride.IsConstant() ? "constant" : "expression", + count.has_value() ? (count->IsConstant() ? "constant" : "expression") + : "unknown"); + + return std::make_unique(die_id, std::move(name), + element_type_ref, std::move(count), + std::move(byte_stride)); } std::unique_ptr diff --git a/lldb/source/Plugins/SymbolFile/DWARF/DWARFUnit.cpp b/lldb/source/Plugins/SymbolFile/DWARF/DWARFUnit.cpp index 8b0fade86f177..54f82af9dbbbb 100644 --- a/lldb/source/Plugins/SymbolFile/DWARF/DWARFUnit.cpp +++ b/lldb/source/Plugins/SymbolFile/DWARF/DWARFUnit.cpp @@ -9,6 +9,8 @@ #include "DWARFUnit.h" #include "lldb/Core/Module.h" +#include "lldb/Core/Value.h" +#include "lldb/Expression/DWARFExpressionList.h" #include "lldb/Symbol/ObjectFile.h" #include "lldb/Utility/LLDBAssert.h" #include "lldb/Utility/StreamString.h" @@ -727,6 +729,187 @@ DWARFUnit::GetDIEBitSizeAndSign(uint64_t relative_die_offset) const { return std::pair{bit_size, sign}; } +llvm::Expected> +DWARFUnit::GetDIELocationExpression( + uint64_t die_offset, bool unit_relative, + std::optional pc_function_offset) const { + // FIXME: the constness has annoying ripple effects. + DWARFUnit *self = const_cast(this); + DWARFDIE die; + if (unit_relative) { + // DW_OP_call2/DW_OP_call4: the offset is relative to the start of this + // unit. + die = self->GetDIE(GetOffset() + die_offset); + } else { + // DW_OP_call_ref: the offset is relative to the start of the .debug_info + // section and may reference a DIE in another unit. + die = self->GetSymbolFileDWARF().DebugInfo().GetDIE( + DIERef::Section::DebugInfo, die_offset); + } + if (!die) + return llvm::createStringError( + "cannot resolve DW_OP_call target DIE at offset 0x%" PRIx64, + die_offset); + + DWARFFormValue form_value; + if (!die.GetDIE()->GetAttributeValue(die.GetCU(), DW_AT_location, form_value, + /*end_attr_offset_ptr=*/nullptr, + /*check_elaborating_dies=*/true)) + // The DIE has no DW_AT_location attribute. Return an empty expression; + // per the DWARF specification the call operation then has no effect. + return std::pair{DataExtractor(), + static_cast(this)}; + + if (!DWARFFormValue::IsBlockForm(form_value.Form())) { + // The DW_AT_location is a location list: which expression describes the + // value depends on the program counter. Parse the list and select the + // entry covering the current PC. + const DWARFUnit *expr_unit = form_value.GetUnit(); + if (!pc_function_offset) + return llvm::createStringError( + "DIE at offset 0x%" PRIx64 " has a location list DW_AT_location, " + "but the current PC is unknown so no entry can be selected", + die_offset); + + // Location list ranges are expressed in the address space of this + // DWARF, which is not necessarily the address space the program runs + // in (for example on Darwin without a dSYM, where the DWARF lives in + // the .o files). Translate the function-relative PC into this space by + // anchoring it at the entry point of the function containing the + // referenced DIE; DIEs whose locations are referenced by DW_OP_call* + // or DW_OP_implicit_pointer describe values of the function being + // executed, so this is the same function the PC offset is relative to. + DWARFDIE func_die = die; + while (func_die && func_die.Tag() != DW_TAG_subprogram) + func_die = func_die.GetParent(); + if (!func_die) + return llvm::createStringError( + "DIE at offset 0x%" PRIx64 " has a location list DW_AT_location " + "but no enclosing DW_TAG_subprogram to anchor the PC against", + die_offset); + const dw_addr_t func_low_pc = + func_die.GetAttributeValueAsAddress(DW_AT_low_pc, LLDB_INVALID_ADDRESS); + if (func_low_pc == LLDB_INVALID_ADDRESS) + return llvm::createStringError( + "the function enclosing the DIE at offset 0x%" PRIx64 + " has no DW_AT_low_pc to anchor the PC against for its location " + "list", + die_offset); + const dw_addr_t pc_file_addr = func_low_pc + *pc_function_offset; + + DataExtractor loc_data = expr_unit->GetLocationData(); + uint64_t loc_offset = form_value.Unsigned(); + if (form_value.Form() == DW_FORM_loclistx) { + std::optional indexed_offset = + const_cast(expr_unit)->GetLoclistOffset(loc_offset); + if (!indexed_offset) + return llvm::createStringError( + "DIE at offset 0x%" PRIx64 " has an unresolvable DW_FORM_loclistx " + "location list index %" PRIu64, + die_offset, loc_offset); + loc_offset = *indexed_offset; + } + if (!loc_data.ValidOffset(loc_offset)) + return llvm::createStringError( + "DIE at offset 0x%" PRIx64 " has a location list offset 0x%" PRIx64 + " outside the location list section", + die_offset, loc_offset); + + loc_data = DataExtractor(loc_data, loc_offset, + loc_data.GetByteSize() - loc_offset); + lldb::ModuleSP module_sp = + GetSymbolFileDWARF().GetObjectFile()->GetModule(); + DWARFExpressionList location_list(module_sp, DWARFExpression(), expr_unit); + if (!expr_unit->ParseDWARFLocationList(loc_data, location_list)) + return llvm::createStringError( + "failed to parse the location list of the DIE at offset 0x%" PRIx64, + die_offset); + + // The list entries hold file address ranges; passing + // LLDB_INVALID_ADDRESS as the function load address makes the lookup + // interpret pc_file_addr in file address terms as well. + const DWARFExpression *expr = location_list.GetExpressionAtAddress( + LLDB_INVALID_ADDRESS, pc_file_addr); + if (!expr) + // No entry covers the current PC: the value is unavailable at this + // point in the program. This is the location list analogue of a + // missing DW_AT_location attribute, so report it the same way, with + // an empty expression. DW_OP_call* then has no effect, which + // compiler-emitted guard expressions (push a fallback; call; branch + // on whether the call pushed a value) rely on, and dereferencing an + // implicit pointer reports the pointee as unavailable. + return std::pair{DataExtractor(), + static_cast(this)}; + + DataExtractor expr_data; + if (!expr->GetExpressionData(expr_data)) + return llvm::createStringError( + "failed to extract the location list expression of the DIE at " + "offset 0x%" PRIx64, + die_offset); + expr_data.SetAddressByteSize(expr_unit->GetAddressByteSize()); + return std::pair{expr_data, + static_cast(expr_unit)}; + } + + const uint8_t *block_data = form_value.BlockData(); + uint64_t block_length = form_value.Unsigned(); + if (!block_data || block_length == 0) + return std::pair{DataExtractor(), + static_cast(this)}; + + // The attribute may have been found on an elaborating DIE + // (DW_AT_specification, DW_AT_abstract_origin or DW_AT_signature), so read + // the expression block from, and evaluate it against, the unit that + // actually contains it. + const DWARFUnit *expr_unit = form_value.GetUnit(); + const DWARFDataExtractor &data = expr_unit->GetData(); + uint64_t block_offset = block_data - data.GetDataStart(); + DataExtractor expr_data(data, block_offset, block_length); + expr_data.SetAddressByteSize(expr_unit->GetAddressByteSize()); + return std::pair{expr_data, + static_cast(expr_unit)}; +} + +llvm::Expected> +DWARFUnit::GetDIEConstValue(uint64_t die_offset) const { + // FIXME: the constness has annoying ripple effects. + DWARFUnit *self = const_cast(this); + DWARFDIE die = self->GetSymbolFileDWARF().DebugInfo().GetDIE( + DIERef::Section::DebugInfo, die_offset); + if (!die) + return llvm::createStringError( + "cannot resolve DIE at .debug_info offset 0x%" PRIx64, die_offset); + + DWARFFormValue form_value; + if (!die.GetDIE()->GetAttributeValue(die.GetCU(), DW_AT_const_value, + form_value, + /*end_attr_offset_ptr=*/nullptr, + /*check_elaborating_dies=*/true)) + return std::nullopt; + + if (DWARFFormValue::IsBlockForm(form_value.Form())) { + const uint8_t *block_data = form_value.BlockData(); + const uint64_t block_length = form_value.Unsigned(); + if (!block_data || block_length == 0) + return llvm::createStringError( + "DIE at .debug_info offset 0x%" PRIx64 + " has an empty DW_AT_const_value block", + die_offset); + return Value(block_data, static_cast(block_length)); + } + + if (const char *str = form_value.AsCString()) + return Value(str, strlen(str) + 1); + + // An integer constant. Only DW_FORM_sdata carries a signedness of its + // own; for the fixed-size data forms signedness would come from the + // type, which is not known here. + if (form_value.Form() == DW_FORM_sdata) + return Value(Scalar(form_value.Signed())); + return Value(Scalar(form_value.Unsigned())); +} + lldb::offset_t DWARFUnit::GetVendorDWARFOpcodeSize(const DataExtractor &data, const lldb::offset_t data_offset, diff --git a/lldb/source/Plugins/SymbolFile/DWARF/DWARFUnit.h b/lldb/source/Plugins/SymbolFile/DWARF/DWARFUnit.h index f55400eeaa448..8306f6f7761d5 100644 --- a/lldb/source/Plugins/SymbolFile/DWARF/DWARFUnit.h +++ b/lldb/source/Plugins/SymbolFile/DWARF/DWARFUnit.h @@ -160,6 +160,18 @@ class DWARFUnit : public DWARFExpression::Delegate, public UserID { llvm::Expected> GetDIEBitSizeAndSign(uint64_t relative_die_offset) const override; + uint8_t GetDWARFOffsetByteSize() const override { + return GetFormParams().getDwarfOffsetByteSize(); + } + + llvm::Expected> + GetDIELocationExpression(uint64_t die_offset, bool unit_relative, + std::optional pc_function_offset) + const override; + + llvm::Expected> + GetDIEConstValue(uint64_t die_offset) const override; + lldb::offset_t GetVendorDWARFOpcodeSize(const DataExtractor &data, const lldb::offset_t data_offset, const uint8_t op) const override; diff --git a/lldb/source/Plugins/SymbolFile/DWARF/SymbolFileDWARF.cpp b/lldb/source/Plugins/SymbolFile/DWARF/SymbolFileDWARF.cpp index c47145b3622c0..d45845c33e9a8 100644 --- a/lldb/source/Plugins/SymbolFile/DWARF/SymbolFileDWARF.cpp +++ b/lldb/source/Plugins/SymbolFile/DWARF/SymbolFileDWARF.cpp @@ -3598,6 +3598,19 @@ VariableSP SymbolFileDWARF::ParseVariableDIE(const SymbolContext &sc, this, unlinked_file_addr); }); scope = eValueTypeVariableThreadLocal; + } else { + // A local's location list entries may also contain DW_OP_addr + // operands holding .o file addresses, e.g. a pointer to a + // statically allocated object described as DW_OP_addr , + // DW_OP_stack_value. Link them into the executable's address + // space, like the static-variable case above does for its + // single expression. + location_list.LinkDWOPAddrOperands( + [this, debug_map_symfile]( + lldb::addr_t unlinked_file_addr) -> lldb::addr_t { + return debug_map_symfile->LinkOSOFileAddress( + this, unlinked_file_addr); + }); } } } @@ -4089,6 +4102,29 @@ SymbolFileDWARF::CollectCallEdges(ModuleSP module, DWARFDIE function_die) { CallSiteParameterArray parameters = CollectCallSiteParameters(module, child); + // The expressions above were built from the raw DWARF, so under a debug + // map any DW_OP_addr operand they contain still holds a .o file address. + // Link such operands into the executable's address space, and point the + // expressions at the linked module so the addresses resolve there when + // they are evaluated. Note that LocationInCallee must stay untouched: it + // is matched byte-for-byte against the callee's DW_OP_entry_value + // sub-expression (also raw DWARF) and is never evaluated. + if (SymbolFileDWARFDebugMap *debug_map_symfile = GetDebugMapSymfile()) { + ModuleSP linked_module_sp = + debug_map_symfile->GetObjectFile()->GetModule(); + const auto link_address = [this](lldb::addr_t file_addr) { + return FixupAddress(file_addr); + }; + for (CallSiteParameter ¶m : parameters) { + param.LocationInCaller.LinkDWOPAddrOperands(link_address); + param.LocationInCaller.SetModule(linked_module_sp); + } + if (call_target) { + call_target->LinkDWOPAddrOperands(link_address); + call_target->SetModule(linked_module_sp); + } + } + std::unique_ptr edge; if (call_origin) { LLDB_LOG(log, diff --git a/lldb/source/Plugins/TypeSystem/OxCaml/CMakeLists.txt b/lldb/source/Plugins/TypeSystem/OxCaml/CMakeLists.txt index 627617d133dba..ba27f64c1b5de 100644 --- a/lldb/source/Plugins/TypeSystem/OxCaml/CMakeLists.txt +++ b/lldb/source/Plugins/TypeSystem/OxCaml/CMakeLists.txt @@ -2,10 +2,12 @@ set(LLVM_LINK_COMPONENTS Support) add_lldb_library(lldbPluginTypeSystemOxCaml PLUGIN TypeSystemOxCaml.cpp + OxCamlDynamicLayoutValue.cpp OxCamlTypes.cpp LINK_LIBS lldbCore + lldbExpression lldbSymbol lldbTarget lldbUtility diff --git a/lldb/source/Plugins/TypeSystem/OxCaml/OxCamlDynamicLayoutValue.cpp b/lldb/source/Plugins/TypeSystem/OxCaml/OxCamlDynamicLayoutValue.cpp new file mode 100644 index 0000000000000..a2cee4771002a --- /dev/null +++ b/lldb/source/Plugins/TypeSystem/OxCaml/OxCamlDynamicLayoutValue.cpp @@ -0,0 +1,181 @@ +//===-- OxCamlDynamicLayoutValue.cpp ----------------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "OxCamlDynamicLayoutValue.h" + +#include "../../Language/OxCaml/LogChannelOxCaml.h" +#include "../../Language/OxCaml/OxCamlAssert.h" +#include "lldb/Core/Value.h" +#include "lldb/Expression/DWARFExpression.h" +#include "lldb/Target/RegisterContext.h" +#include "lldb/Target/StackFrame.h" +#include "lldb/Utility/Log.h" +#include "lldb/Utility/Scalar.h" +#include "llvm/Support/Error.h" + +using namespace lldb; +using namespace lldb_private; + +// ============================================================================ +// OxCamlDynamicLayoutValue +// ============================================================================ + +OxCamlDynamicLayoutValue +OxCamlDynamicLayoutValue::MakeConstant(uint64_t value, + OxCamlLayoutValueKind kind) { + return OxCamlDynamicLayoutValue(kind, InitialStackLayout::Empty, value, + std::nullopt); +} + +OxCamlDynamicLayoutValue +OxCamlDynamicLayoutValue::MakeExpression(DWARFExpressionList expression, + OxCamlLayoutValueKind kind, + InitialStackLayout initial_stack) { + return OxCamlDynamicLayoutValue(kind, initial_stack, 0, + std::move(expression)); +} + +static Value MakeObjectAddressValue(lldb::addr_t address) { + Value v; + v.GetScalar() = address; + v.SetValueType(Value::ValueType::Scalar); + return v; +} + +// Wrap [error] with an OxCaml-channel log entry and return it. All evaluation +// failures (DWARF errors, missing object address, kind mismatch) flow through +// here so every call site gets consistent logging on the user-visible channel. +// The caller is responsible for emitting a user-visible formatter marker +// (OXCAML_EMIT_MARKER) when the failure surfaces in formatter output. +static llvm::Error LogAndReturnError(llvm::Error error) { + std::string message = llvm::toString(std::move(error)); + if (Log *log = GetLog(OxCamlLog::UserVisibleErrors)) + LLDB_LOG(log, "OxCamlDynamicLayoutValue evaluation failed: {0}", message); + return llvm::createStringError(llvm::inconvertibleErrorCode(), message); +} + +llvm::Expected OxCamlDynamicLayoutValue::Evaluate( + const OxCamlFormatContext &context, + std::optional object_address) const { + if (IsConstant()) { + if (m_kind == OxCamlLayoutValueKind::Location) + return OxCamlLayoutResult::MakeObjectRelativeByteOffset(m_constant); + return OxCamlLayoutResult::MakeScalar(m_kind, m_constant); + } + + ExecutionContext exe_ctx(context.exe_ctx_ref); + + Value initial_value; + const Value *initial_value_ptr = nullptr; + if (m_initial_stack == InitialStackLayout::ObjectAddress) { + if (!object_address.has_value()) + return LogAndReturnError(llvm::createStringError( + "OxCaml dynamic layout expression needs the OCaml object address " + "but none is available")); + initial_value = MakeObjectAddressValue(*object_address); + initial_value_ptr = &initial_value; + } + + Value object_address_value; + const Value *object_address_ptr = nullptr; + if (object_address.has_value()) { + object_address_value = MakeObjectAddressValue(*object_address); + object_address_ptr = &object_address_value; + } + + // Pass the current frame's RegisterContext so DWARF expressions that mention + // registers (DW_OP_breg*, DW_OP_regx, etc.) can be evaluated correctly. + RegisterContext *reg_ctx = nullptr; + if (StackFrame *frame = exe_ctx.GetFramePtr()) + reg_ctx = frame->GetRegisterContext().get(); + + llvm::Expected result = + m_expression->Evaluate(&exe_ctx, reg_ctx, + /*func_load_addr=*/LLDB_INVALID_ADDRESS, + initial_value_ptr, object_address_ptr); + if (!result) + return LogAndReturnError(result.takeError()); + + if (m_kind == OxCamlLayoutValueKind::Location) { + if (result->GetValueType() != Value::ValueType::LoadAddress) + return LogAndReturnError(llvm::createStringError( + "OxCaml location-valued expression did not produce a load address " + "(value-type %d); a trailing DW_OP_stack_value would make a " + "location-valued attribute scalar by mistake", + static_cast(result->GetValueType()))); + return OxCamlLayoutResult::MakeLoadAddress( + result->GetScalar().ULongLong(LLDB_INVALID_ADDRESS)); + } + + if (result->GetValueType() != Value::ValueType::Scalar) + return LogAndReturnError(llvm::createStringError( + "OxCaml scalar-valued expression did not produce an implicit scalar " + "(value-type %d); did the expression omit DW_OP_stack_value?", + static_cast(result->GetValueType()))); + return OxCamlLayoutResult::MakeScalar(m_kind, + result->GetScalar().ULongLong()); +} + +// ============================================================================ +// OxCamlLayoutResult +// ============================================================================ + +OxCamlLayoutResult OxCamlLayoutResult::MakeScalar(OxCamlLayoutValueKind kind, + uint64_t scalar) { + OX_ASSERT(kind != OxCamlLayoutValueKind::Location, + "MakeScalar requires a scalar kind (got {0})", + static_cast(kind)); + return OxCamlLayoutResult(kind, LocationStorage::ObjectRelativeByteOffset, + scalar, LLDB_INVALID_ADDRESS, 0); +} + +OxCamlLayoutResult +OxCamlLayoutResult::MakeLoadAddress(lldb::addr_t load_address) { + return OxCamlLayoutResult(OxCamlLayoutValueKind::Location, + LocationStorage::LoadAddress, 0, load_address, 0); +} + +OxCamlLayoutResult +OxCamlLayoutResult::MakeObjectRelativeByteOffset(uint64_t byte_offset) { + return OxCamlLayoutResult(OxCamlLayoutValueKind::Location, + LocationStorage::ObjectRelativeByteOffset, 0, + LLDB_INVALID_ADDRESS, byte_offset); +} + +uint64_t OxCamlLayoutResult::GetScalar() const { + OX_ASSERT(m_kind != OxCamlLayoutValueKind::Location, + "GetScalar called on a location-valued result (kind={0})", + static_cast(m_kind)); + return m_scalar; +} + +OxCamlLayoutResult::LocationStorage +OxCamlLayoutResult::GetLocationStorage() const { + OX_ASSERT(m_kind == OxCamlLayoutValueKind::Location, + "GetLocationStorage called on a non-location result (kind={0})", + static_cast(m_kind)); + return m_location_storage; +} + +lldb::addr_t OxCamlLayoutResult::GetLoadAddress() const { + OX_ASSERT(m_kind == OxCamlLayoutValueKind::Location && + m_location_storage == LocationStorage::LoadAddress, + "GetLoadAddress called on a non-load-address result (kind={0}, " + "storage={1})", + static_cast(m_kind), static_cast(m_location_storage)); + return m_load_address; +} + +uint64_t OxCamlLayoutResult::GetObjectRelativeByteOffset() const { + OX_ASSERT(m_kind == OxCamlLayoutValueKind::Location && + m_location_storage == LocationStorage::ObjectRelativeByteOffset, + "GetObjectRelativeByteOffset called on a non-object-relative " + "result (kind={0}, storage={1})", + static_cast(m_kind), static_cast(m_location_storage)); + return m_object_relative_byte_offset; +} diff --git a/lldb/source/Plugins/TypeSystem/OxCaml/OxCamlDynamicLayoutValue.h b/lldb/source/Plugins/TypeSystem/OxCaml/OxCamlDynamicLayoutValue.h new file mode 100644 index 0000000000000..8296ddd39e47e --- /dev/null +++ b/lldb/source/Plugins/TypeSystem/OxCaml/OxCamlDynamicLayoutValue.h @@ -0,0 +1,157 @@ +//===-- OxCamlDynamicLayoutValue.h ------------------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef LLDB_SOURCE_PLUGINS_TYPESYSTEM_OXCAML_OXCAMLDYNAMICLAYOUTVALUE_H +#define LLDB_SOURCE_PLUGINS_TYPESYSTEM_OXCAML_OXCAMLDYNAMICLAYOUTVALUE_H + +#include "../../Language/OxCaml/OxCamlAssert.h" +#include "lldb/Expression/DWARFExpressionList.h" +#include "lldb/Target/ExecutionContext.h" +#include "lldb/lldb-private.h" +#include "llvm/Support/Error.h" +#include +#include + +namespace lldb_private { + +class OxCamlLayoutResult; + +/// Describes the unit of a dynamic-layout value's result. +/// +/// - ScalarBits/ScalarBytes/ScalarElements: the value is a scalar quantity +/// whose unit is fixed by the attribute it came from (e.g. a byte size, a +/// bit offset, an element count). Scalar exprlocs end with +/// DW_OP_stack_value. +/// - Location: the value identifies a memory location. Constants are +/// object-relative byte offsets; exprlocs evaluate to a load address. +enum class OxCamlLayoutValueKind { + ScalarBits, + ScalarBytes, + ScalarElements, + Location, +}; + +/// Session-invariant context for the formatter: process, frame, byte order, +/// pointer size. Set once at the entry point and threaded by const reference +/// through recursive calls. Per-recursion state (e.g. the OCaml object +/// address inherited from an enclosing FormatPointer) is passed as a +/// separate parameter. +struct OxCamlFormatContext { + lldb::ProcessSP process_sp; + ExecutionContextRef exe_ctx_ref; + lldb::ByteOrder byte_order; + uint32_t address_byte_size; + /// The module the value's debug info lives in; needed to evaluate the + /// DWARF descriptions of optimized-out values (implicit pointers). + lldb::ModuleSP module_sp; +}; + +/// A layout fact that may be a constant or a DWARF expression. +/// +/// The wrapper is generic over the result's value-kind and the expression's +/// initial-stack convention. Parsing helpers fix the (kind, initial_stack) +/// pair per DWARF attribute, since DWARF carries no signal that varies per +/// occurrence. +class OxCamlDynamicLayoutValue { +public: + /// What the DWARF evaluator should pre-load on the expression stack before + /// running the expression. + enum class InitialStackLayout { Empty, ObjectAddress }; + + /// Build a constant layout value. For Location kind, the constant is an + /// object-relative byte offset. + static OxCamlDynamicLayoutValue MakeConstant(uint64_t value, + OxCamlLayoutValueKind kind); + + static OxCamlDynamicLayoutValue + MakeExpression(DWARFExpressionList expression, OxCamlLayoutValueKind kind, + InitialStackLayout initial_stack); + + bool IsConstant() const { return !m_expression.has_value(); } + OxCamlLayoutValueKind GetKind() const { return m_kind; } + InitialStackLayout GetInitialStack() const { return m_initial_stack; } + + /// Returns the constant value. Asserts when IsConstant() is false, because + /// reading the constant out of an expression-valued layout would indicate a + /// bug in the caller. + uint64_t GetConstant() const { + OX_ASSERT(IsConstant(), + "GetConstant() called on a non-constant OxCamlDynamicLayoutValue " + "(kind={0})", + static_cast(m_kind)); + return m_constant; + } + + /// Evaluate the layout value. Constants wrap the stored constant. + /// Expressions invoke the LLDB DWARF evaluator with [object_address] as + /// the raw OCaml object pointer; evaluation fails cleanly when the + /// expression needs an object address (via the initial-stack convention + /// or DW_OP_push_object_address) but none is supplied. + llvm::Expected + Evaluate(const OxCamlFormatContext &context, + std::optional object_address) const; + +private: + OxCamlDynamicLayoutValue(OxCamlLayoutValueKind kind, + InitialStackLayout initial_stack, uint64_t constant, + std::optional expression) + : m_kind(kind), m_initial_stack(initial_stack), m_constant(constant), + m_expression(std::move(expression)) {} + + OxCamlLayoutValueKind m_kind; + InitialStackLayout m_initial_stack; + uint64_t m_constant = 0; + std::optional m_expression; +}; + +/// Tagged result of evaluating an OxCamlDynamicLayoutValue. +/// +/// The kind drives which accessor is valid: +/// - ScalarBits/ScalarBytes/ScalarElements: GetScalar() returns the value in +/// the indicated unit. +/// - Location: GetLocationStorage() indicates whether the result is an +/// absolute LoadAddress or an ObjectRelativeByteOffset still to be combined +/// with the OCaml object pointer. +/// +/// Misuse is a debug assertion, not a runtime fallback. +class OxCamlLayoutResult { +public: + enum class LocationStorage { LoadAddress, ObjectRelativeByteOffset }; + + static OxCamlLayoutResult MakeScalar(OxCamlLayoutValueKind kind, + uint64_t scalar); + static OxCamlLayoutResult MakeLoadAddress(lldb::addr_t load_address); + static OxCamlLayoutResult MakeObjectRelativeByteOffset(uint64_t byte_offset); + + OxCamlLayoutValueKind GetKind() const { return m_kind; } + + uint64_t GetScalar() const; + + LocationStorage GetLocationStorage() const; + lldb::addr_t GetLoadAddress() const; + uint64_t GetObjectRelativeByteOffset() const; + +private: + OxCamlLayoutResult(OxCamlLayoutValueKind kind, + LocationStorage location_storage, uint64_t scalar, + lldb::addr_t load_address, + uint64_t object_relative_byte_offset) + : m_kind(kind), m_location_storage(location_storage), m_scalar(scalar), + m_load_address(load_address), + m_object_relative_byte_offset(object_relative_byte_offset) {} + + OxCamlLayoutValueKind m_kind; + LocationStorage m_location_storage; + uint64_t m_scalar; + lldb::addr_t m_load_address; + uint64_t m_object_relative_byte_offset; +}; + +} // namespace lldb_private + +#endif // LLDB_SOURCE_PLUGINS_TYPESYSTEM_OXCAML_OXCAMLDYNAMICLAYOUTVALUE_H diff --git a/lldb/source/Plugins/TypeSystem/OxCaml/OxCamlTypes.h b/lldb/source/Plugins/TypeSystem/OxCaml/OxCamlTypes.h index c36ab1d26942e..51e8b3a000b85 100644 --- a/lldb/source/Plugins/TypeSystem/OxCaml/OxCamlTypes.h +++ b/lldb/source/Plugins/TypeSystem/OxCaml/OxCamlTypes.h @@ -10,6 +10,7 @@ #define LLDB_SOURCE_PLUGINS_TYPESYSTEM_OXCAML_OXCAMLTYPES_H #include "../../Language/OxCaml/OxCamlHelpers.h" +#include "OxCamlDynamicLayoutValue.h" #include "lldb/Utility/Reference.h" #include "lldb/lldb-private.h" #include @@ -49,8 +50,6 @@ class OxCamlType { virtual uint64_t GetByteSize() const = 0; - virtual int64_t GetPointerAdjustmentOffset() const { return 0; } - virtual std::string GetDefaultDisplayName() const = 0; protected: @@ -188,20 +187,27 @@ class OxCamlPointerType : public OxCamlType { class OxCamlArrayType : public OxCamlType { Reference *m_element_type_ref; - std::optional m_count; - uint64_t m_stride; + /// DW_AT_count, in elements. The storage is optional because DWARF permits + /// unbounded arrays, but OxCaml compiler-emitted array DIEs must have a + /// subrange count expression that reads the runtime block wosize. + std::optional m_count; + /// DW_AT_byte_stride, in bytes. OxCaml compiler output emits this explicitly. + OxCamlDynamicLayoutValue m_stride; public: OxCamlArrayType(lldb::user_id_t die_id, std::optional name, Reference *element_type_ref, - std::optional count, uint64_t stride) + std::optional count, + OxCamlDynamicLayoutValue stride) : OxCamlType(Array, die_id, std::move(name)), - m_element_type_ref(element_type_ref), m_count(count), m_stride(stride) { - } + m_element_type_ref(element_type_ref), m_count(std::move(count)), + m_stride(std::move(stride)) {} OxCamlType *GetElementType() const; - std::optional GetCount() const { return m_count; } - uint64_t GetStride() const { return m_stride; } + const std::optional &GetCount() const { + return m_count; + } + const OxCamlDynamicLayoutValue &GetStride() const { return m_stride; } uint64_t GetByteSize() const override { return formatters::oxcaml::helpers::constants::WORD_SIZE; } @@ -213,9 +219,14 @@ class OxCamlArrayType : public OxCamlType { struct OxCamlMember { std::optional name; Reference *type_ref; - uint64_t data_member_location; - std::optional bit_offset; - std::optional bit_size; + /// Object-address-relative location of this member. Constants are byte + /// offsets from the raw OCaml object pointer; exprlocs evaluate to a load + /// address using the pre-pushed pointer convention. + OxCamlDynamicLayoutValue location; + /// Optional bit-field offset and size. Both are scalar quantities in bits + /// (constants or empty-stack exprlocs). + std::optional bit_offset; + std::optional bit_size; bool is_artificial = false; bool IsBitField() const { @@ -250,34 +261,33 @@ class OxCamlVariantPart { class OxCamlStructureType : public OxCamlType { private: - uint64_t m_byte_size; + /// Dynamic size (ScalarBytes or ScalarBits, constant or expression). The + /// unit is recorded by [m_size.GetKind()]; callers that need bytes must + /// convert. [m_static_size_fallback] (always in bytes) is used by LLDB + /// APIs that need a static size without an execution context. + OxCamlDynamicLayoutValue m_size; + uint64_t m_static_size_fallback; std::vector m_members; std::vector m_variant_parts; - // Custom base offset from DW_AT_ocaml_offset_record_from_pointer. This is - // specific to the DWARF encoding used by the OxCaml compiler. It supports an - // ad-hoc attribute (see DW_AT_ocaml_offset_record_from_pointer in - // DWARFASTParserOxCaml.cpp) that specifies a byte offset to apply when - // dealing with a pointer to a structure. This allows one to factor in the - // header field of a block when the actual pointer points to the first entry - // in the block. - int64_t m_base_offset; - public: OxCamlStructureType(lldb::user_id_t die_id, std::optional name, - uint64_t byte_size, std::vector members, - std::vector variant_parts = {}, - int64_t base_offset = 0) - : OxCamlType(Structure, die_id, std::move(name)), m_byte_size(byte_size), + OxCamlDynamicLayoutValue size, + uint64_t static_size_fallback, + std::vector members, + std::vector variant_parts = {}) + : OxCamlType(Structure, die_id, std::move(name)), + m_size(std::move(size)), + m_static_size_fallback(static_size_fallback), m_members(std::move(members)), - m_variant_parts(std::move(variant_parts)), m_base_offset(base_offset) {} + m_variant_parts(std::move(variant_parts)) {} - uint64_t GetByteSize() const override { return m_byte_size; } + uint64_t GetByteSize() const override { return m_static_size_fallback; } + const OxCamlDynamicLayoutValue &GetDynamicSize() const { return m_size; } const std::vector &GetMembers() const { return m_members; } const std::vector &GetVariantParts() const { return m_variant_parts; } - int64_t GetPointerAdjustmentOffset() const override { return m_base_offset; } bool IsOCamlVariant() const; bool IsTuple() const; diff --git a/lldb/source/Plugins/TypeSystem/OxCaml/TypeSystemOxCaml.cpp b/lldb/source/Plugins/TypeSystem/OxCaml/TypeSystemOxCaml.cpp index f01a2b98ca644..e07775887b067 100644 --- a/lldb/source/Plugins/TypeSystem/OxCaml/TypeSystemOxCaml.cpp +++ b/lldb/source/Plugins/TypeSystem/OxCaml/TypeSystemOxCaml.cpp @@ -338,8 +338,15 @@ void TypeSystemOxCaml::Dump(llvm::raw_ostream &output, llvm::StringRef filter) { auto *at = static_cast(actual_type); output << " (array of " << at->GetElementType()->GetDisplayName(); if (at->GetCount().has_value()) - output << ", count=" << at->GetCount().value(); - output << ", stride=" << at->GetStride() << " bytes)"; + output << ", count=" + << (at->GetCount()->IsConstant() + ? std::to_string(at->GetCount()->GetConstant()) + : std::string("")); + output << ", stride=" + << (at->GetStride().IsConstant() + ? std::to_string(at->GetStride().GetConstant()) + : std::string("")) + << " bytes)"; } break; case OxCamlType::Placeholder: output << " (placeholder - parsing in progress)"; diff --git a/lldb/source/Symbol/Function.cpp b/lldb/source/Symbol/Function.cpp index 6114eccd935ee..95e1a78d346b5 100644 --- a/lldb/source/Symbol/Function.cpp +++ b/lldb/source/Symbol/Function.cpp @@ -201,7 +201,8 @@ DirectCallEdge::DirectCallEdge(const char *symbol_name, lazy_callee.symbol_name = symbol_name; } -Function *DirectCallEdge::GetCallee(ModuleList &images, ExecutionContext &) { +Function *DirectCallEdge::GetCallee(ModuleList &images, ExecutionContext &, + const EntryValueResolutionContext *) { ParseSymbolFileAndResolve(images); assert(resolved && "Did not resolve lazy callee"); return lazy_callee.def; @@ -216,14 +217,15 @@ IndirectCallEdge::IndirectCallEdge(DWARFExpressionList call_target, std::move(parameters)), call_target(std::move(call_target)) {} -Function *IndirectCallEdge::GetCallee(ModuleList &images, - ExecutionContext &exe_ctx) { +Function * +IndirectCallEdge::GetCallee(ModuleList &images, ExecutionContext &exe_ctx, + const EntryValueResolutionContext *entry_value_ctx) { Log *log = GetLog(LLDBLog::Step); Status error; llvm::Expected callee_addr_val = call_target.Evaluate( &exe_ctx, exe_ctx.GetRegisterContext(), LLDB_INVALID_ADDRESS, /*initial_value_ptr=*/nullptr, - /*object_address_ptr=*/nullptr); + /*object_address_ptr=*/nullptr, entry_value_ctx); if (!callee_addr_val) { LLDB_LOG_ERROR(log, callee_addr_val.takeError(), "IndirectCallEdge: Could not evaluate expression: {0}"); diff --git a/lldb/source/Target/ABI.cpp b/lldb/source/Target/ABI.cpp index b86fef6bf03e8..7d5a1b4b0cf51 100644 --- a/lldb/source/Target/ABI.cpp +++ b/lldb/source/Target/ABI.cpp @@ -125,6 +125,7 @@ ValueObjectSP ABI::GetReturnValueObject(Thread &thread, CompilerType &ast_type, return {}; case Value::ValueType::HostAddress: case Value::ValueType::FileAddress: + case Value::ValueType::ImplicitPointer: // we odon't do anything with these for now break; case Value::ValueType::Scalar: diff --git a/lldb/source/Target/Language.cpp b/lldb/source/Target/Language.cpp index 86754c251cd93..dcabad349e3aa 100644 --- a/lldb/source/Target/Language.cpp +++ b/lldb/source/Target/Language.cpp @@ -17,7 +17,10 @@ #include "lldb/Symbol/SymbolFile.h" #include "lldb/Symbol/TypeList.h" #include "lldb/Target/Target.h" +#include "lldb/Utility/DataExtractor.h" +#include "lldb/Utility/Status.h" #include "lldb/Utility/Stream.h" +#include "lldb/ValueObject/ValueObject.h" #include "llvm/BinaryFormat/Dwarf.h" #include "llvm/Support/Threading.h" @@ -502,6 +505,20 @@ DumpValueObjectOptions::DeclPrintingHelper Language::GetDeclPrintingHelper() { return nullptr; } +llvm::Expected +Language::GetDefaultExternalFormatterInput(ValueObject &valobj, + std::vector &data) { + DataExtractor extractor; + Status error; + valobj.GetData(extractor, error); + if (error.Fail()) + return llvm::createStringError("cannot get value data: %s", + error.AsCString()); + const uint8_t *bytes = extractor.GetDataStart(); + data.assign(bytes, bytes + extractor.GetByteSize()); + return std::string(valobj.GetDisplayTypeName().AsCString("")); +} + LazyBool Language::IsLogicalTrue(ValueObject &valobj, Status &error) { return eLazyBoolCalculate; } diff --git a/lldb/source/Target/StackFrameList.cpp b/lldb/source/Target/StackFrameList.cpp index 16cd2548c2784..310346744b0aa 100644 --- a/lldb/source/Target/StackFrameList.cpp +++ b/lldb/source/Target/StackFrameList.cpp @@ -177,7 +177,13 @@ static void FindInterveningFrames(Function &begin, Function &end, } // The first callee may not be resolved, or there may be nothing to fill in. - Function *first_callee = first_edge->GetCallee(images, exe_ctx); + // No explicit entry-value resolution context applies here: a DW_OP_entry_value + // in this regular call's target would refer to the begin function's own + // entry, for which we have no call edge (it would require walking to the + // begin frame's caller, which is unavailable while the frame list is being + // built). Such a target therefore simply stays unresolved, as before. + Function *first_callee = + first_edge->GetCallee(images, exe_ctx, /*entry_value_ctx=*/nullptr); if (!first_callee) { LLDB_LOG(log, "Could not resolve callee"); return; @@ -207,12 +213,20 @@ static void FindInterveningFrames(Function &begin, Function &end, void search(CallEdge &first_edge, Function &first_callee, CallSequence &path) { - dfs(first_edge, first_callee); + // The first callee was reached by a regular (non-tail) call from the + // frame in \ref context, which is a live frame. A DW_OP_entry_value in + // one of the first callee's outgoing edges therefore refers to the first + // callee's entry and is resolved against that live caller via + // \ref first_edge. + EntryValueResolutionContext first_ctx{&first_edge, context.GetFramePtr(), + /*outer=*/nullptr}; + dfs(first_edge, first_callee, &first_ctx); if (!ambiguous) path = std::move(solution_path); } - void dfs(CallEdge ¤t_edge, Function &callee) { + void dfs(CallEdge ¤t_edge, Function &callee, + const EntryValueResolutionContext *callee_ctx) { // Found a path to the target function. if (&callee == end) { if (solution_path.empty()) @@ -234,14 +248,21 @@ static void FindInterveningFrames(Function &begin, Function &end, // Search the calls made from this callee. active_path.push_back(CallDescriptor{&callee}); for (const auto &edge : callee.GetTailCallingEdges()) { - Function *next_callee = edge->GetCallee(images, context); + // A DW_OP_entry_value in this edge's indirect target refers to the + // entry of \p callee; \ref callee_ctx says how to resolve it. The + // function reached through this edge is itself frameless (it was + // tail-called), so if its own entry values are needed deeper in the + // search they chain back through \ref callee_ctx. + Function *next_callee = edge->GetCallee(images, context, callee_ctx); if (!next_callee) continue; std::tie(active_path.back().address_type, active_path.back().address) = edge->GetCallerAddress(callee, target); - dfs(*edge, *next_callee); + EntryValueResolutionContext next_ctx{ + edge.get(), /*caller_frame=*/nullptr, callee_ctx}; + dfs(*edge, *next_callee, &next_ctx); if (ambiguous) return; } diff --git a/lldb/source/ValueObject/ValueObject.cpp b/lldb/source/ValueObject/ValueObject.cpp index a6fb3fad65251..3fa4b90d813ff 100644 --- a/lldb/source/ValueObject/ValueObject.cpp +++ b/lldb/source/ValueObject/ValueObject.cpp @@ -311,6 +311,12 @@ const char *ValueObject::GetLocationAsCStringImpl(const Value &value, m_location_str = "scalar"; break; + case Value::ValueType::ImplicitPointer: + // The pointer was optimized away; it has no numeric value (which is + // distinct from having the value zero). + m_location_str = "implicit pointer"; + break; + case Value::ValueType::LoadAddress: case Value::ValueType::FileAddress: case Value::ValueType::HostAddress: { @@ -801,6 +807,10 @@ bool ValueObject::SetData(DataExtractor &data, Status &error) { case Value::ValueType::Invalid: error = Status::FromErrorString("invalid location"); return false; + case Value::ValueType::ImplicitPointer: + error = Status::FromErrorString( + "cannot modify the value of an implicit pointer"); + return false; case Value::ValueType::Scalar: { Status set_error = m_value.GetScalar().SetValueFromData(data, encoding, byte_size); @@ -1607,6 +1617,7 @@ ValueObject::GetAddressOf(bool scalar_is_load_address) { switch (m_value.GetValueType()) { case Value::ValueType::Invalid: + case Value::ValueType::ImplicitPointer: return {}; case Value::ValueType::Scalar: if (scalar_is_load_address) { @@ -1632,6 +1643,11 @@ ValueObject::AddrAndType ValueObject::GetPointerValue() { switch (m_value.GetValueType()) { case Value::ValueType::Invalid: return {}; + case Value::ValueType::ImplicitPointer: + // The pointer was optimized away and has no numeric value. Returning + // the invalid address (and not zero) keeps this case distinct from a + // pointer whose value is a null pointer. + return {}; case Value::ValueType::Scalar: return {m_value.GetScalar().ULongLong(LLDB_INVALID_ADDRESS), GetAddressTypeOfChildren()}; @@ -1728,6 +1744,10 @@ bool ValueObject::SetValueFromCString(const char *value_str, Status &error) { case Value::ValueType::Invalid: error = Status::FromErrorString("invalid location"); return false; + case Value::ValueType::ImplicitPointer: + error = Status::FromErrorString( + "cannot modify the value of an implicit pointer"); + return false; case Value::ValueType::FileAddress: case Value::ValueType::Scalar: break; diff --git a/lldb/source/ValueObject/ValueObjectChild.cpp b/lldb/source/ValueObject/ValueObjectChild.cpp index e29634c2f3322..96fade253e25c 100644 --- a/lldb/source/ValueObject/ValueObjectChild.cpp +++ b/lldb/source/ValueObject/ValueObjectChild.cpp @@ -145,6 +145,10 @@ bool ValueObjectChild::UpdateValue() { switch (m_value.GetValueType()) { case Value::ValueType::Invalid: break; + case Value::ValueType::ImplicitPointer: + m_error = Status::FromErrorString( + "parent value is an implicit pointer with no child storage"); + break; case Value::ValueType::LoadAddress: case Value::ValueType::FileAddress: case Value::ValueType::HostAddress: { diff --git a/lldb/source/ValueObject/ValueObjectMemory.cpp b/lldb/source/ValueObject/ValueObjectMemory.cpp index 3d8d80c6ec480..9ff3356e0ad52 100644 --- a/lldb/source/ValueObject/ValueObjectMemory.cpp +++ b/lldb/source/ValueObject/ValueObjectMemory.cpp @@ -179,6 +179,10 @@ bool ValueObjectMemory::UpdateValue() { case Value::ValueType::Invalid: m_error = Status::FromErrorString("Invalid value"); return false; + case Value::ValueType::ImplicitPointer: + // Values read from the target's memory can never be implicit pointers. + m_error = Status::FromErrorString("unexpected implicit pointer value"); + return false; case Value::ValueType::Scalar: // The variable value is in the Scalar value inside the m_value. We can // point our m_data right to it. diff --git a/lldb/source/ValueObject/ValueObjectVariable.cpp b/lldb/source/ValueObject/ValueObjectVariable.cpp index 12a84f9f2ed74..599c133874342 100644 --- a/lldb/source/ValueObject/ValueObjectVariable.cpp +++ b/lldb/source/ValueObject/ValueObjectVariable.cpp @@ -201,6 +201,15 @@ bool ValueObjectVariable::UpdateValue() { case Value::ValueType::Invalid: m_error = Status::FromErrorString("invalid value"); break; + case Value::ValueType::ImplicitPointer: + // The variable is a pointer that was optimized away, but the value + // it would point at is described by another DIE (the m_value + // carries that reference). The pointer itself has no bytes to read; + // note that its value being unavailable is distinct from it being + // a null pointer. Consumers such as language formatters can + // materialize the pointed-at value with + // DWARFExpression::DereferenceImplicitPointer(). + break; case Value::ValueType::Scalar: // The variable value is in the Scalar value inside the m_value. We can // point our m_data right to it. @@ -262,6 +271,7 @@ void ValueObjectVariable::DoUpdateChildrenAddressType(ValueObject &valobj) { switch (value_type) { case Value::ValueType::Invalid: + case Value::ValueType::ImplicitPointer: break; case Value::ValueType::FileAddress: // If this type is a pointer, then its children will be considered load diff --git a/lldb/unittests/Expression/DWARFExpressionTest.cpp b/lldb/unittests/Expression/DWARFExpressionTest.cpp index fdc9bfae1876c..4a3a60d5e24eb 100644 --- a/lldb/unittests/Expression/DWARFExpressionTest.cpp +++ b/lldb/unittests/Expression/DWARFExpressionTest.cpp @@ -343,6 +343,405 @@ TEST(DWARFExpression, DW_OP_convert) { llvm::Failed()); } +TEST(DWARFExpression, DW_OP_call) { + // Two compile units. CU0 contains a variable DIE whose DW_AT_location + // pushes 0x2211. CU1, the unit the expressions below are evaluated in, + // contains: + // - a variable DIE whose location pushes 0x47 (CU-relative 0x0e) + // - a variable DIE without DW_AT_location (CU-relative 0x13) + // - a variable DIE whose location calls itself (CU-relative 0x15) + // - a variable DIE whose location adds 1 to the value on top of the + // stack (CU-relative 0x1c) + // DW_OP_call2/DW_OP_call4 offsets are relative to the current unit (CU1), + // whereas the DW_OP_call_ref offset is relative to the start of + // .debug_info and here resolves to the DIE in CU0. + const char *yamldata = R"( +--- !ELF +FileHeader: + Class: ELFCLASS64 + Data: ELFDATA2LSB + Type: ET_EXEC + Machine: EM_386 +DWARF: + debug_abbrev: + - Table: + - Code: 0x00000001 + Tag: DW_TAG_compile_unit + Children: DW_CHILDREN_yes + Attributes: + - Attribute: DW_AT_language + Form: DW_FORM_data2 + - Code: 0x00000002 + Tag: DW_TAG_variable + Children: DW_CHILDREN_no + Attributes: + - Attribute: DW_AT_location + Form: DW_FORM_exprloc + - Code: 0x00000003 + Tag: DW_TAG_variable + Children: DW_CHILDREN_no + Attributes: + - Attribute: DW_AT_const_value + Form: DW_FORM_data1 + debug_info: + - Version: 4 + AddrSize: 8 + AbbrevTableID: 0 + AbbrOffset: 0x0 + Entries: + - AbbrCode: 0x00000001 + Values: + - Value: 0x000000000000000C + # 0x0000000e: DW_AT_location = DW_OP_const2u(0x2211) + - AbbrCode: 0x00000002 + Values: + - BlockData: [ 0x0a, 0x11, 0x22 ] + - AbbrCode: 0x00000000 + - Version: 4 + AddrSize: 8 + AbbrevTableID: 0 + AbbrOffset: 0x0 + Entries: + - AbbrCode: 0x00000001 + Values: + - Value: 0x000000000000000C + # CU-relative 0x0e: DW_AT_location = DW_OP_const2u(0x0047) + - AbbrCode: 0x00000002 + Values: + - BlockData: [ 0x0a, 0x47, 0x00 ] + # CU-relative 0x13: no DW_AT_location. + - AbbrCode: 0x00000003 + Values: + - Value: 0x0000000000000000 + # CU-relative 0x15: DW_AT_location = DW_OP_call4(0x15), i.e. a DIE + # whose location expression calls itself. + - AbbrCode: 0x00000002 + Values: + - BlockData: [ 0x99, 0x15, 0x00, 0x00, 0x00 ] + # CU-relative 0x1c: DW_AT_location = DW_OP_lit1, DW_OP_plus. + - AbbrCode: 0x00000002 + Values: + - BlockData: [ 0x31, 0x22 ] + - AbbrCode: 0x00000000 +)"; + + // Offsets of the DIEs above, relative to the start of CU1. + uint8_t offs_const_0x47 = 0x0e; + uint8_t offs_no_location = 0x13; + uint8_t offs_self_call = 0x15; + uint8_t offs_add_one = 0x1c; + // Offset of CU0's constant DIE, relative to the start of .debug_info. + uint8_t offs_cu0_const_0x2211 = 0x0e; + + DWARFExpressionTester t(yamldata, /*cu_index=*/1); + ASSERT_TRUE((bool)t.GetDwarfUnit()); + + bool is_signed = true; + bool not_signed = false; + + // Values left on the stack by the called expression are visible to the + // calling expression. + EXPECT_THAT_EXPECTED(t.Eval({DW_OP_call2, offs_const_0x47, 0x00, // + DW_OP_stack_value}), + llvm::HasValue(GetScalar(64, 0x47, not_signed))); + + EXPECT_THAT_EXPECTED( + t.Eval({DW_OP_call4, offs_const_0x47, 0x00, 0x00, 0x00, // + DW_OP_stack_value}), + llvm::HasValue(GetScalar(64, 0x47, not_signed))); + + // DW_OP_call_ref resolves its offset in .debug_info, finding the DIE in + // CU0 (whereas the same unit-relative offset names the 0x47 DIE in CU1). + EXPECT_THAT_EXPECTED( + t.Eval({DW_OP_call_ref, offs_cu0_const_0x2211, 0x00, 0x00, 0x00, // + DW_OP_stack_value}), + llvm::HasValue(GetScalar(64, 0x2211, not_signed))); + + // Calling a DIE that has no DW_AT_location attribute has no effect. + EXPECT_THAT_EXPECTED(t.Eval({DW_OP_lit4, // + DW_OP_call4, offs_no_location, 0, 0, 0, // + DW_OP_stack_value}), + llvm::HasValue(GetScalar(32, 4, is_signed))); + + // Values on the stack at the time of the call may be used as parameters + // by the called expression. + EXPECT_THAT_EXPECTED(t.Eval({DW_OP_lit4, // + DW_OP_call4, offs_add_one, 0, 0, 0, // + DW_OP_stack_value}), + llvm::HasValue(GetScalar(64, 5, is_signed))); + + // + // Errors. + // + + // A self-referencing call must be diagnosed rather than recurse forever. + EXPECT_THAT_ERROR( + t.Eval({DW_OP_call4, offs_self_call, 0, 0, 0}).takeError(), + llvm::Failed()); + + // No DIE at the given offset (it points into the unit header). + EXPECT_THAT_ERROR(t.Eval({DW_OP_call4, 0x01, 0, 0, 0}).takeError(), + llvm::Failed()); + + // No compile unit. + EXPECT_THAT_ERROR( + Evaluate({DW_OP_call4, offs_const_0x47, 0, 0, 0}).takeError(), + llvm::Failed()); +} + +TEST(DWARFExpression, DW_OP_implicit_pointer) { + // A compile unit containing (offsets are relative to the start of + // .debug_info, which coincides with unit-relative offsets here): + // - 0x0e: variable A whose location is the scalar 0x2211 + // (DW_OP_const2u, DW_OP_stack_value) + // - 0x14: variable B whose location is the four implicit bytes + // aa bb cc dd (DW_OP_implicit_value) + // - 0x1c: variable C without DW_AT_location or DW_AT_const_value + // - 0x1e: variable D whose location is an implicit pointer to B + // - 0x26: variable E whose location is the scalar 0 + // (DW_OP_lit0, DW_OP_stack_value) + // - 0x2a: variable F whose value is the integer constant 42 + // (DW_AT_const_value) + // - 0x2c: variable G whose value is the constant bytes 10 20 + // (DW_AT_const_value block) + const char *yamldata = R"( +--- !ELF +FileHeader: + Class: ELFCLASS64 + Data: ELFDATA2LSB + Type: ET_EXEC + Machine: EM_386 +DWARF: + debug_abbrev: + - Table: + - Code: 0x00000001 + Tag: DW_TAG_compile_unit + Children: DW_CHILDREN_yes + Attributes: + - Attribute: DW_AT_language + Form: DW_FORM_data2 + - Code: 0x00000002 + Tag: DW_TAG_variable + Children: DW_CHILDREN_no + Attributes: + - Attribute: DW_AT_location + Form: DW_FORM_exprloc + - Code: 0x00000003 + Tag: DW_TAG_variable + Children: DW_CHILDREN_no + Attributes: + - Attribute: DW_AT_const_value + Form: DW_FORM_data1 + - Code: 0x00000004 + Tag: DW_TAG_variable + Children: DW_CHILDREN_no + Attributes: + - Attribute: DW_AT_decl_file + Form: DW_FORM_data1 + - Code: 0x00000005 + Tag: DW_TAG_variable + Children: DW_CHILDREN_no + Attributes: + - Attribute: DW_AT_const_value + Form: DW_FORM_block1 + debug_info: + - Version: 4 + AddrSize: 8 + AbbrevTableID: 0 + AbbrOffset: 0x0 + Entries: + - AbbrCode: 0x00000001 + Values: + - Value: 0x000000000000000C + # 0x0e: A: DW_AT_location = DW_OP_const2u(0x2211), DW_OP_stack_value + - AbbrCode: 0x00000002 + Values: + - BlockData: [ 0x0a, 0x11, 0x22, 0x9f ] + # 0x14: B: DW_AT_location = DW_OP_implicit_value(aa bb cc dd) + - AbbrCode: 0x00000002 + Values: + - BlockData: [ 0x9e, 0x04, 0xaa, 0xbb, 0xcc, 0xdd ] + # 0x1c: C: neither DW_AT_location nor DW_AT_const_value. + - AbbrCode: 0x00000004 + Values: + - Value: 0x0000000000000000 + # 0x1e: D: DW_AT_location = DW_OP_implicit_pointer(0x14, 0) + - AbbrCode: 0x00000002 + Values: + - BlockData: [ 0xa0, 0x14, 0x00, 0x00, 0x00, 0x00 ] + # 0x26: E: DW_AT_location = DW_OP_lit0, DW_OP_stack_value + - AbbrCode: 0x00000002 + Values: + - BlockData: [ 0x30, 0x9f ] + # 0x2a: F: DW_AT_const_value = 42 + - AbbrCode: 0x00000003 + Values: + - Value: 0x000000000000002A + # 0x2c: G: DW_AT_const_value = block(10 20) + - AbbrCode: 0x00000005 + Values: + - BlockData: [ 0x10, 0x20 ] + - AbbrCode: 0x00000000 +)"; + + DWARFExpressionTester t(yamldata, /*cu_index=*/0); + ASSERT_TRUE((bool)t.GetDwarfUnit()); + + auto evaluate = [&](llvm::ArrayRef expr) -> llvm::Expected { + DataExtractor extractor(expr.data(), expr.size(), lldb::eByteOrderLittle, + /*addr_size*/ 4); + return DWARFExpression::Evaluate( + /*exe_ctx*/ nullptr, /*reg_ctx*/ nullptr, t.GetModule(), extractor, + t.GetDwarfUnit(), lldb::eRegisterKindLLDB, + /*initial_value_ptr*/ nullptr, /*object_address_ptr*/ nullptr); + }; + auto dereference = [&](const Value &v) -> llvm::Expected { + return DWARFExpression::DereferenceImplicitPointer( + v, /*exe_ctx*/ nullptr, /*reg_ctx*/ nullptr, t.GetModule()); + }; + + uint8_t offs_scalar = 0x0e; // A + uint8_t offs_bytes = 0x14; // B + uint8_t offs_no_location = 0x1c; // C + uint8_t offs_chained = 0x1e; // D + uint8_t offs_zero = 0x26; // E + uint8_t offs_const_scalar = 0x2a; // F + uint8_t offs_const_block = 0x2c; // G + + // Evaluating DW_OP_implicit_pointer yields an implicit-pointer value + // carrying the DIE reference and byte offset. The pointer itself has no + // numeric value: asking for one yields the caller's fail value, never + // something that could be mistaken for a real pointer. + llvm::Expected result = + evaluate({DW_OP_implicit_pointer, offs_scalar, 0, 0, 0, 2}); + ASSERT_THAT_EXPECTED(result, llvm::Succeeded()); + ASSERT_EQ(result->GetValueType(), Value::ValueType::ImplicitPointer); + ASSERT_TRUE(result->GetImplicitPointer().has_value()); + EXPECT_EQ(result->GetImplicitPointer()->die_offset, (uint64_t)offs_scalar); + EXPECT_EQ(result->GetImplicitPointer()->byte_offset, 2); + EXPECT_EQ(result->GetScalar().ULongLong(LLDB_INVALID_ADDRESS), + LLDB_INVALID_ADDRESS); + + // The legacy GNU extension behaves identically. + result = evaluate({DW_OP_GNU_implicit_pointer, offs_scalar, 0, 0, 0, 0}); + ASSERT_THAT_EXPECTED(result, llvm::Succeeded()); + EXPECT_EQ(result->GetValueType(), Value::ValueType::ImplicitPointer); + + // Dereferencing materializes the pointed-at value: here a scalar. + result = evaluate({DW_OP_implicit_pointer, offs_scalar, 0, 0, 0, 0}); + ASSERT_THAT_EXPECTED(result, llvm::Succeeded()); + llvm::Expected pointee = dereference(*result); + ASSERT_THAT_EXPECTED(pointee, llvm::Succeeded()); + EXPECT_EQ(pointee->GetValueType(), Value::ValueType::Scalar); + EXPECT_EQ(pointee->GetScalar().ULongLong(), 0x2211u); + + // ...here a blob of bytes living in the debugger's memory, with the + // implicit pointer's byte offset applied. + result = evaluate({DW_OP_implicit_pointer, offs_bytes, 0, 0, 0, 1}); + ASSERT_THAT_EXPECTED(result, llvm::Succeeded()); + pointee = dereference(*result); + ASSERT_THAT_EXPECTED(pointee, llvm::Succeeded()); + ASSERT_EQ(pointee->GetValueType(), Value::ValueType::HostAddress); + ASSERT_EQ(pointee->GetBuffer().GetByteSize(), 3u); + EXPECT_EQ(pointee->GetBuffer().GetBytes()[0], 0xbb); + EXPECT_EQ(pointee->GetBuffer().GetBytes()[2], 0xdd); + + // A chain of implicit pointers: D's own location is an implicit pointer, + // so dereferencing yields another implicit pointer, which in turn + // dereferences to B's bytes. + result = evaluate({DW_OP_implicit_pointer, offs_chained, 0, 0, 0, 0}); + ASSERT_THAT_EXPECTED(result, llvm::Succeeded()); + pointee = dereference(*result); + ASSERT_THAT_EXPECTED(pointee, llvm::Succeeded()); + ASSERT_EQ(pointee->GetValueType(), Value::ValueType::ImplicitPointer); + ASSERT_TRUE(pointee->GetImplicitPointer().has_value()); + EXPECT_EQ(pointee->GetImplicitPointer()->die_offset, (uint64_t)offs_bytes); + pointee = dereference(*pointee); + ASSERT_THAT_EXPECTED(pointee, llvm::Succeeded()); + ASSERT_EQ(pointee->GetValueType(), Value::ValueType::HostAddress); + ASSERT_EQ(pointee->GetBuffer().GetByteSize(), 4u); + EXPECT_EQ(pointee->GetBuffer().GetBytes()[0], 0xaa); + + // A pointee that legitimately evaluates to zero is a success... + result = evaluate({DW_OP_implicit_pointer, offs_zero, 0, 0, 0, 0}); + ASSERT_THAT_EXPECTED(result, llvm::Succeeded()); + pointee = dereference(*result); + ASSERT_THAT_EXPECTED(pointee, llvm::Succeeded()); + EXPECT_EQ(pointee->GetValueType(), Value::ValueType::Scalar); + EXPECT_EQ(pointee->GetScalar().ULongLong(/*fail_value=*/1), 0u); + + // The pointed-at value may also be described by a DW_AT_const_value + // attribute instead of a DW_AT_location: an integer constant... + result = evaluate({DW_OP_implicit_pointer, offs_const_scalar, 0, 0, 0, 0}); + ASSERT_THAT_EXPECTED(result, llvm::Succeeded()); + pointee = dereference(*result); + ASSERT_THAT_EXPECTED(pointee, llvm::Succeeded()); + EXPECT_EQ(pointee->GetValueType(), Value::ValueType::Scalar); + EXPECT_EQ(pointee->GetScalar().ULongLong(), 42u); + + // ...or a block of bytes, with the byte offset applied. + result = evaluate({DW_OP_implicit_pointer, offs_const_block, 0, 0, 0, 1}); + ASSERT_THAT_EXPECTED(result, llvm::Succeeded()); + pointee = dereference(*result); + ASSERT_THAT_EXPECTED(pointee, llvm::Succeeded()); + ASSERT_EQ(pointee->GetValueType(), Value::ValueType::HostAddress); + ASSERT_EQ(pointee->GetBuffer().GetByteSize(), 1u); + EXPECT_EQ(pointee->GetBuffer().GetBytes()[0], 0x20); + + // ...whereas an unavailable pointee (neither DW_AT_location nor + // DW_AT_const_value on the referenced DIE) is an error, never a zero + // value. + result = evaluate({DW_OP_implicit_pointer, offs_no_location, 0, 0, 0, 0}); + ASSERT_THAT_EXPECTED(result, llvm::Succeeded()); + EXPECT_THAT_EXPECTED(dereference(*result), llvm::Failed()); + + // A DIE offset that does not resolve is likewise an error. + result = evaluate({DW_OP_implicit_pointer, 0x01, 0, 0, 0, 0}); + ASSERT_THAT_EXPECTED(result, llvm::Succeeded()); + EXPECT_THAT_EXPECTED(dereference(*result), llvm::Failed()); + + // A byte offset cannot be applied to a scalar pointee. + result = evaluate({DW_OP_implicit_pointer, offs_scalar, 0, 0, 0, 2}); + ASSERT_THAT_EXPECTED(result, llvm::Succeeded()); + EXPECT_THAT_EXPECTED(dereference(*result), llvm::Failed()); + + // A composite (DW_OP_piece) value with an implicit-pointer piece: the + // composite's buffer carries placeholder bytes for the piece, and the + // piece's actual value is recorded in the implicit-pointer table. This is + // how an optimized-out boxed value is described: each piece is one field + // of the block, and pointer-valued fields may be implicit pointers. + result = evaluate({DW_OP_lit1, DW_OP_stack_value, DW_OP_piece, 4, // + DW_OP_implicit_pointer, offs_bytes, 0, 0, 0, 0, // + DW_OP_piece, 8}); + ASSERT_THAT_EXPECTED(result, llvm::Succeeded()); + ASSERT_EQ(result->GetValueType(), Value::ValueType::HostAddress); + ASSERT_EQ(result->GetBuffer().GetByteSize(), 12u); + EXPECT_EQ(result->GetBuffer().GetBytes()[0], 1); + ASSERT_EQ(result->GetImplicitPointerPieces().size(), 1u); + // The second field's range is in the table; the first field's is not. + EXPECT_EQ(result->FindImplicitPointerPiece(0, 4), nullptr); + const Value::ImplicitPointerPiece *piece = + result->FindImplicitPointerPiece(4, 8); + ASSERT_NE(piece, nullptr); + EXPECT_EQ(piece->pointee.die_offset, (uint64_t)offs_bytes); + EXPECT_EQ(piece->pointee.byte_offset, 0); + + // The recorded piece dereferences like any other implicit pointer. + Value piece_pointer; + piece_pointer.SetImplicitPointer(piece->pointee.delegate, + piece->pointee.die_offset, + piece->pointee.byte_offset); + pointee = dereference(piece_pointer); + ASSERT_THAT_EXPECTED(pointee, llvm::Succeeded()); + ASSERT_EQ(pointee->GetValueType(), Value::ValueType::HostAddress); + ASSERT_EQ(pointee->GetBuffer().GetByteSize(), 4u); + EXPECT_EQ(pointee->GetBuffer().GetBytes()[0], 0xaa); + + // Without a compile unit the operation cannot be evaluated. + EXPECT_THAT_EXPECTED(Evaluate({DW_OP_implicit_pointer, 0x0e, 0, 0, 0, 0}), + llvm::Failed()); +} + TEST(DWARFExpression, DW_OP_stack_value) { EXPECT_THAT_EXPECTED(Evaluate({DW_OP_stack_value}), llvm::Failed()); } diff --git a/llvm/include/llvm/DWARFLinker/AddressesMap.h b/llvm/include/llvm/DWARFLinker/AddressesMap.h index e2215c70dc34e..f1ca823f48274 100644 --- a/llvm/include/llvm/DWARFLinker/AddressesMap.h +++ b/llvm/include/llvm/DWARFLinker/AddressesMap.h @@ -55,6 +55,20 @@ class AddressesMap { virtual std::optional getSubprogramRelocAdjustment(const DWARFDie &DIE, bool Verbose) = 0; + /// Maps the object-file address \p ObjectAddress carried by a DW_OP_addr + /// operand inside a *location list* expression to its address in the linked + /// binary. Location-list sections (.debug_loc/.debug_loclists) are not + /// covered by applyValidRelocs (which only handles .debug_info), so the + /// address operands of their expressions have to be relocated explicitly + /// when the list is cloned. + /// \returns the linked address, or std::nullopt if \p ObjectAddress does not + /// correspond to a known live symbol. The default implementation performs no + /// mapping. + virtual std::optional + getExprOpAddressLinkedAddress(uint64_t ObjectAddress) { + return std::nullopt; + } + // Returns the library install name associated to the AddessesMap. virtual std::optional getLibraryInstallName() = 0; diff --git a/llvm/include/llvm/DWARFLinker/Classic/DWARFLinker.h b/llvm/include/llvm/DWARFLinker/Classic/DWARFLinker.h index 5b9535380aebf..7581ad626a39b 100644 --- a/llvm/include/llvm/DWARFLinker/Classic/DWARFLinker.h +++ b/llvm/include/llvm/DWARFLinker/Classic/DWARFLinker.h @@ -671,11 +671,39 @@ class LLVM_ABI DWARFLinker : public DWARFLinkerBase { const DWARFFile &File, CompileUnit &Unit); - /// Clone a DWARF expression that may be referencing another DIE. + /// A DIE reference embedded in a cloned DWARF expression. DIE offsets are + /// only final after every unit is cloned, so the reference operand is + /// emitted as a patchable value and fixed up later (see cloneExpression() + /// and CompileUnit::fixupForwardReferences()). + struct ExpressionDIEReference { + /// Byte offset of the reference operand within the cloned expression. + uint64_t BufferOffset; + /// Size of the operand in bytes (2, 4 or 8). + uint8_t Size; + /// (Possibly placeholder) clone of the referenced DIE. + DIE *RefDie; + /// Compile unit owning \p RefDie. + const CompileUnit *RefUnit; + /// True for absolute .debug_info references (DW_OP_call_ref, + /// DW_OP_implicit_pointer); false for CU-relative ones (DW_OP_call2/4). + bool SectionRelative; + }; + + /// Clone a DWARF expression that may be referencing another DIE. Any DIE + /// references found in the expression are appended to \p References so the + /// caller can emit them as patchable values. + /// + /// When \p RelocateExprLocAddr is true the expression is part of a location + /// list (.debug_loc/.debug_loclists), whose address operands are not + /// relocated by applyValidRelocs; any DW_OP_addr operand is then translated + /// to its linked address via the debug map. For exprlocs in .debug_info it + /// must be false, as those operands are relocated by applyValidRelocs. void cloneExpression(DataExtractor &Data, DWARFExpression Expression, const DWARFFile &File, CompileUnit &Unit, SmallVectorImpl &OutputBuffer, - int64_t AddrRelocAdjustment, bool IsLittleEndian); + int64_t AddrRelocAdjustment, bool IsLittleEndian, + SmallVectorImpl &References, + bool RelocateExprLocAddr = false); /// Clone an attribute referencing another DIE and add /// it to \p Die. diff --git a/llvm/include/llvm/DWARFLinker/Classic/DWARFLinkerCompileUnit.h b/llvm/include/llvm/DWARFLinker/Classic/DWARFLinkerCompileUnit.h index 714badfabf89c..4939a9ffef9a4 100644 --- a/llvm/include/llvm/DWARFLinker/Classic/DWARFLinkerCompileUnit.h +++ b/llvm/include/llvm/DWARFLinker/Classic/DWARFLinkerCompileUnit.h @@ -196,9 +196,12 @@ class CompileUnit { /// Keep track of a forward reference to DIE \p Die in \p RefUnit by \p /// Attr. The attribute should be fixed up later to point to the absolute /// offset of \p Die in the debug_info section or to the canonical offset of - /// \p Ctxt if it is non-null. + /// \p Ctxt if it is non-null. When \p SectionRelative is false the fixup + /// instead uses the offset of \p Die relative to the start of \p RefUnit, + /// as needed by CU-relative DWARF expression references (DW_OP_call2/call4). LLVM_ABI void noteForwardReference(DIE *Die, const CompileUnit *RefUnit, - DeclContext *Ctxt, PatchLocation Attr); + DeclContext *Ctxt, PatchLocation Attr, + bool SectionRelative = true); /// Apply all fixups recorded by noteForwardReference(). LLVM_ABI void fixupForwardReferences(); @@ -297,8 +300,8 @@ class CompileUnit { /// The offsets for the attributes in this array couldn't be set while /// cloning because for cross-cu forward references the target DIE's offset /// isn't known you emit the reference attribute. - std::vector< - std::tuple> + std::vector> ForwardDIEReferences; /// The ranges in that map are the PC ranges for functions in this unit, diff --git a/llvm/lib/DWARFLinker/Classic/DWARFLinker.cpp b/llvm/lib/DWARFLinker/Classic/DWARFLinker.cpp index 222dc88098102..df5e64e6589a2 100644 --- a/llvm/lib/DWARFLinker/Classic/DWARFLinker.cpp +++ b/llvm/lib/DWARFLinker/Classic/DWARFLinker.cpp @@ -630,6 +630,10 @@ unsigned DWARFLinker::shouldKeepDIE(AddressesMap &RelocMgr, const DWARFDie &DIE, case dwarf::DW_TAG_base_type: // DWARF Expressions may reference basic types, but scanning them // is expensive. Basic types are tiny, so just keep all of them. + case dwarf::DW_TAG_dwarf_procedure: + // DWARF procedures are the targets of DW_OP_call2/call4/call_ref. Those + // calls can appear in location lists, which the keep-analysis does not + // scan, so keep all procedures (they are small) to be safe. case dwarf::DW_TAG_imported_module: case dwarf::DW_TAG_imported_declaration: case dwarf::DW_TAG_imported_unit: @@ -738,6 +742,57 @@ void DWARFLinker::markODRCanonicalDie(const DWARFDie &Die, CompileUnit &CU) { Info.Ctxt->setHasCanonicalDIE(); } +/// Collect DIEs referenced by DWARF expression operands inside the location +/// expression \p Val and append them to \p ReferencedDIEs. DW_OP_call2/call4, +/// DW_OP_call_ref and DW_OP_implicit_pointer (and its GNU predecessor) all +/// carry a reference to another DIE; those targets must be kept so that +/// DIECloner::cloneExpression() can later remap the reference to its new +/// location instead of emitting a dangling offset. +static void collectExpressionReferencedDIEs( + const DWARFFormValue &Val, CompileUnit &CU, const UnitListTy &Units, + SmallVectorImpl> &ReferencedDIEs) { + std::optional> Expr = Val.getAsBlock(); + if (!Expr || Expr->empty()) + return; + + DWARFUnit &Unit = CU.getOrigUnit(); + DataExtractor Data(toStringRef(*Expr), Unit.getContext().isLittleEndian(), + Unit.getAddressByteSize()); + DWARFExpression Expression(Data, Unit.getAddressByteSize(), + Unit.getFormParams().Format); + + for (const DWARFExpression::Operation &Op : Expression) { + if (Op.isError()) + break; + + uint64_t RefOffset; + switch (Op.getCode()) { + case dwarf::DW_OP_call2: + case dwarf::DW_OP_call4: + // Compilation-unit-relative reference (DW_FORM_ref2/ref4-like). + RefOffset = Unit.getOffset() + Op.getRawOperand(0); + break; + case dwarf::DW_OP_call_ref: + case dwarf::DW_OP_implicit_pointer: + case dwarf::DW_OP_GNU_implicit_pointer: + // Section-relative reference (DW_FORM_ref_addr-like). + RefOffset = Op.getRawOperand(0); + break; + default: + continue; + } + + CompileUnit *RefUnit = getUnitForOffset(Units, RefOffset); + if (!RefUnit) + continue; + DWARFDie RefDie = RefUnit->getOrigUnit().getDIEForOffset(RefOffset); + if (!RefDie || RefDie.isNULL()) + continue; + RefUnit->getInfo(RefDie).Prune = false; + ReferencedDIEs.emplace_back(RefDie, *RefUnit); + } +} + /// Look at DIEs referenced by the given DIE and decide whether they should be /// kept. All DIEs referenced though attributes should be kept. void DWARFLinker::lookForRefDIEsToKeep( @@ -757,8 +812,18 @@ void DWARFLinker::lookForRefDIEsToKeep( DWARFFormValue Val(AttrSpec.Form); if (!Val.isFormClass(DWARFFormValue::FC_Reference) || AttrSpec.Attr == dwarf::DW_AT_sibling) { - DWARFFormValue::skipValue(AttrSpec.Form, Data, &Offset, - Unit.getFormParams()); + // A location expression may reference other DIEs through DW_OP_call* and + // DW_OP_implicit_pointer; those targets must be kept too. Everything else + // is skipped. + if (DWARFAttribute::mayHaveLocationExpr(AttrSpec.Attr) && + (Val.isFormClass(DWARFFormValue::FC_Block) || + Val.isFormClass(DWARFFormValue::FC_Exprloc))) { + Val.extractValue(Data, &Offset, Unit.getFormParams(), &Unit); + collectExpressionReferencedDIEs(Val, CU, Units, ReferencedDIEs); + } else { + DWARFFormValue::skipValue(AttrSpec.Form, Data, &Offset, + Unit.getFormParams()); + } continue; } @@ -1156,11 +1221,51 @@ unsigned DWARFLinker::DIECloner::cloneDieReferenceAttribute( void DWARFLinker::DIECloner::cloneExpression( DataExtractor &Data, DWARFExpression Expression, const DWARFFile &File, CompileUnit &Unit, SmallVectorImpl &OutputBuffer, - int64_t AddrRelocAdjustment, bool IsLittleEndian) { + int64_t AddrRelocAdjustment, bool IsLittleEndian, + SmallVectorImpl &References, + bool RelocateExprLocAddr) { using Encoding = DWARFExpression::Operation::Encoding; uint8_t OrigAddressByteSize = Unit.getOrigUnit().getAddressByteSize(); + // Record a reference to the DIE at original .debug_info offset \p AbsOffset + // and reserve \p Size zero bytes for the operand in the output buffer. The + // referenced DIE's clone may not exist yet (forward reference), so a + // placeholder is created if needed and the final offset is filled in by + // CompileUnit::fixupForwardReferences() once all DIEs are cloned. With + // \p SectionRelative the patched value is an absolute .debug_info offset + // (DW_FORM_ref_addr-like); otherwise it is relative to the start of the + // referenced unit (DW_FORM_ref2/ref4-like). + auto AppendDIEReference = [&](uint64_t AbsOffset, unsigned Size, + bool SectionRelative, const Twine &Kind) { + DIE *RefClone = nullptr; + CompileUnit *RefUnit = getUnitForOffset(CompileUnits, AbsOffset); + DWARFDie RefDie = + RefUnit ? RefUnit->getOrigUnit().getDIEForOffset(AbsOffset) : DWARFDie(); + if (!RefUnit || !RefDie || RefDie.isNULL()) { + Linker.reportWarning(Kind + " references an unknown DIE.", File); + } else { + CompileUnit::DIEInfo &Info = RefUnit->getInfo(RefDie); + if (!Info.Keep) { + Linker.reportWarning(Kind + " references a DIE that was not kept.", + File); + } else { + if (!Info.Clone) { + // Forward (or not-yet-cloned) reference: create the placeholder the + // real clone will reuse, exactly like cloneDieReferenceAttribute(). + Info.UnclonedReference = true; + Info.Clone = DIE::get(DIEAlloc, dwarf::Tag(RefDie.getTag())); + } + RefClone = Info.Clone; + } + } + + if (RefClone) + References.push_back({OutputBuffer.size(), static_cast(Size), + RefClone, RefUnit, SectionRelative}); + OutputBuffer.append(Size, uint8_t(0)); + }; + uint64_t OpOffset = 0; for (auto &Op : Expression) { auto Desc = Op.getDescription(); @@ -1266,6 +1371,59 @@ void DWARFLinker::DIECloner::cloneExpression( } } else Linker.reportWarning("cannot read DW_OP_constx operand.", File); + } else if (RelocateExprLocAddr && Op.getCode() == dwarf::DW_OP_addr) { + // A DW_OP_addr operand inside a location list carries the object-file + // address of a symbol (e.g. a statically-allocated OCaml closure). + // Location lists are not relocated by applyValidRelocs, so translate the + // operand to its linked address via the debug map here. (For exprlocs in + // .debug_info, RelocateExprLocAddr is false and the operand is relocated + // by applyValidRelocs instead, so this must not run for them.) + uint64_t ObjectAddress = Op.getRawOperand(0); + std::optional LinkedAddress; + if (File.Addresses) + LinkedAddress = + File.Addresses->getExprOpAddressLinkedAddress(ObjectAddress); + if (LinkedAddress) { + OutputBuffer.push_back(dwarf::DW_OP_addr); + uint64_t Address = *LinkedAddress; + if (IsLittleEndian != sys::IsLittleEndianHost) + sys::swapByteOrder(Address); + ArrayRef AddressBytes( + reinterpret_cast(&Address), OrigAddressByteSize); + OutputBuffer.append(AddressBytes.begin(), AddressBytes.end()); + } else { + // Unknown address; copy the operation unmodified rather than emit a + // bogus value. + StringRef Bytes = Data.getData().slice(OpOffset, Op.getEndOffset()); + OutputBuffer.append(Bytes.begin(), Bytes.end()); + } + } else if (Op.getCode() == dwarf::DW_OP_call_ref || + Op.getCode() == dwarf::DW_OP_implicit_pointer || + Op.getCode() == dwarf::DW_OP_GNU_implicit_pointer) { + // These take a .debug_info-section-relative reference to a DIE + // (interpreted like DW_FORM_ref_addr) as their first operand. + // DW_OP_implicit_pointer additionally carries a trailing SLEB128 byte + // offset, copied unmodified. The reference is recorded for deferred + // patching so it is correct even for forward references. The operand + // width matches what the decoder consumed (DWARF32 vs DWARF64). + unsigned RefSize = Op.getOperandEndOffset(0) - (OpOffset + 1); + OutputBuffer.push_back(Op.getCode()); + AppendDIEReference(Op.getRawOperand(0), RefSize, /*SectionRelative=*/true, + dwarf::OperationEncodingString(Op.getCode())); + StringRef Tail = + Data.getData().slice(Op.getOperandEndOffset(0), Op.getEndOffset()); + OutputBuffer.append(Tail.begin(), Tail.end()); + } else if (Op.getCode() == dwarf::DW_OP_call2 || + Op.getCode() == dwarf::DW_OP_call4) { + // DW_OP_call2/call4 take a compilation-unit-relative reference to a DIE + // (interpreted like DW_FORM_ref2/ref4), encoded as a fixed 2- or 4-byte + // value. Record it for deferred patching to the cloned DIE's new + // unit-relative offset. + unsigned RefSize = Op.getOperandEndOffset(0) - (OpOffset + 1); + uint64_t AbsOffset = Op.getRawOperand(0) + Unit.getOrigUnit().getOffset(); + OutputBuffer.push_back(Op.getCode()); + AppendDIEReference(AbsOffset, RefSize, /*SectionRelative=*/false, + dwarf::OperationEncodingString(Op.getCode())); } else { // Copy over everything else unmodified. StringRef Bytes = Data.getData().slice(OpOffset, Op.getEndOffset()); @@ -1297,6 +1455,7 @@ unsigned DWARFLinker::DIECloner::cloneBlockAttribute( // If the block is a DWARF Expression, clone it into the temporary // buffer using cloneExpression(), otherwise copy the data directly. SmallVector Buffer; + SmallVector References; ArrayRef Bytes = *Val.getAsBlock(); if (DWARFAttribute::mayHaveLocationExpr(AttrSpec.Attr) && (Val.isFormClass(DWARFFormValue::FC_Block) || @@ -1306,12 +1465,32 @@ unsigned DWARFLinker::DIECloner::cloneBlockAttribute( DWARFExpression Expr(Data, OrigUnit.getAddressByteSize(), OrigUnit.getFormParams().Format); cloneExpression(Data, Expr, File, Unit, Buffer, - Unit.getInfo(InputDIE).AddrAdjust, IsLittleEndian); + Unit.getInfo(InputDIE).AddrAdjust, IsLittleEndian, + References); Bytes = Buffer; } - for (auto Byte : Bytes) - Attr->addValue(DIEAlloc, static_cast(0), - dwarf::DW_FORM_data1, DIEInteger(Byte)); + // Emit the expression bytes as DIE values. DIE references collected by + // cloneExpression() are emitted as a single patchable integer and recorded + // for fixup once DIE offsets are final; all other bytes are emitted as + // individual DW_FORM_data1 values. + size_t NextRef = 0; + for (size_t I = 0, E = Bytes.size(); I != E;) { + if (NextRef < References.size() && References[NextRef].BufferOffset == I) { + const ExpressionDIEReference &Ref = References[NextRef++]; + dwarf::Form Form = Ref.Size == 2 ? dwarf::DW_FORM_data2 + : Ref.Size == 4 ? dwarf::DW_FORM_data4 + : dwarf::DW_FORM_data8; + auto Patch = Attr->addValue(DIEAlloc, static_cast(0), + Form, DIEInteger(0)); + Unit.noteForwardReference(Ref.RefDie, Ref.RefUnit, /*Ctxt=*/nullptr, Patch, + Ref.SectionRelative); + I += Ref.Size; + } else { + Attr->addValue(DIEAlloc, static_cast(0), + dwarf::DW_FORM_data1, DIEInteger(Bytes[I])); + ++I; + } + } // FIXME: If DIEBlock and DIELoc just reuses the Size field of // the DIE class, this "if" could be replaced by @@ -2748,11 +2927,35 @@ uint64_t DWARFLinker::DIECloner::cloneAllCompileUnits( DWARFUnit &OrigUnit = CurrentUnit->getOrigUnit(); DataExtractor Data(SrcBytes, IsLittleEndian, OrigUnit.getAddressByteSize()); + SmallVector References; cloneExpression(Data, DWARFExpression(Data, OrigUnit.getAddressByteSize(), OrigUnit.getFormParams().Format), File, *CurrentUnit, OutBytes, RelocAdjustment, - IsLittleEndian); + IsLittleEndian, References, + /*RelocateExprLocAddr=*/true); + // Location lists are written straight into the section and cannot be + // patched afterwards, so resolve any DIE references now. The current + // unit (and all earlier units) have already been fully cloned, so the + // referenced DIE offsets are final. A reference into a not-yet-cloned + // later unit is left unresolved, with a warning. + for (const ExpressionDIEReference &Ref : References) { + uint64_t TargetOffset = Ref.RefDie->getOffset(); + if (!TargetOffset) { + Linker.reportWarning("DWARF expression in a location list " + "references a DIE in a later unit; reference " + "left unresolved.", + File); + continue; + } + if (Ref.SectionRelative) + TargetOffset += Ref.RefUnit->getStartOffset(); + for (unsigned I = 0; I != Ref.Size; ++I) { + unsigned Shift = IsLittleEndian ? I : Ref.Size - 1 - I; + OutBytes[Ref.BufferOffset + I] = + (TargetOffset >> (8 * Shift)) & 0xff; + } + } }; generateUnitLocations(*CurrentUnit, File, ProcessExpr); emitDebugAddrSection(*CurrentUnit, DwarfVersion); diff --git a/llvm/lib/DWARFLinker/Classic/DWARFLinkerCompileUnit.cpp b/llvm/lib/DWARFLinker/Classic/DWARFLinkerCompileUnit.cpp index 027363aa394e0..44a9fe492cc44 100644 --- a/llvm/lib/DWARFLinker/Classic/DWARFLinkerCompileUnit.cpp +++ b/llvm/lib/DWARFLinker/Classic/DWARFLinkerCompileUnit.cpp @@ -136,8 +136,9 @@ uint64_t CompileUnit::computeNextUnitOffset(uint16_t DwarfVersion) { /// Keep track of a forward cross-cu reference from this unit /// to \p Die that lives in \p RefUnit. void CompileUnit::noteForwardReference(DIE *Die, const CompileUnit *RefUnit, - DeclContext *Ctxt, PatchLocation Attr) { - ForwardDIEReferences.emplace_back(Die, RefUnit, Ctxt, Attr); + DeclContext *Ctxt, PatchLocation Attr, + bool SectionRelative) { + ForwardDIEReferences.emplace_back(Die, RefUnit, Ctxt, Attr, SectionRelative); } void CompileUnit::fixupForwardReferences() { @@ -146,14 +147,20 @@ void CompileUnit::fixupForwardReferences() { const CompileUnit *RefUnit; PatchLocation Attr; DeclContext *Ctxt; - std::tie(RefDie, RefUnit, Ctxt, Attr) = Ref; + bool SectionRelative; + std::tie(RefDie, RefUnit, Ctxt, Attr, SectionRelative) = Ref; if (Ctxt && Ctxt->hasCanonicalDIE()) { assert(Ctxt->getCanonicalDIEOffset() && "Canonical die offset is not set"); Attr.set(Ctxt->getCanonicalDIEOffset()); } else { assert(RefDie->getOffset() && "Referenced die offset is not set"); - Attr.set(RefDie->getOffset() + RefUnit->getStartOffset()); + // CU-relative references (DW_OP_call2/call4 in a DWARF expression) use + // the offset of the DIE within its unit; everything else is an absolute + // .debug_info offset. + Attr.set(SectionRelative + ? RefDie->getOffset() + RefUnit->getStartOffset() + : RefDie->getOffset()); } } } diff --git a/llvm/tools/dsymutil/DwarfLinkerForBinary.cpp b/llvm/tools/dsymutil/DwarfLinkerForBinary.cpp index 138c5d0a513ed..b77128e9832fb 100644 --- a/llvm/tools/dsymutil/DwarfLinkerForBinary.cpp +++ b/llvm/tools/dsymutil/DwarfLinkerForBinary.cpp @@ -591,7 +591,54 @@ void DwarfLinkerForBinary::copySwiftReflectionMetadata( } } +void DwarfLinkerForBinary::buildMainBinarySymbolMap(const DebugMap &Map) { + if (Map.getBinaryPath().empty()) + return; + + auto ObjectEntry = BinHolder.getObjectEntry(Map.getBinaryPath()); + if (!ObjectEntry) { + consumeError(ObjectEntry.takeError()); + return; + } + + auto Object = + ObjectEntry->getObjectAs(Map.getTriple()); + if (!Object) { + consumeError(Object.takeError()); + return; + } + + for (const object::SymbolRef &Sym : Object->symbols()) { + Expected FlagsOrErr = Sym.getFlags(); + if (!FlagsOrErr) { + consumeError(FlagsOrErr.takeError()); + continue; + } + // Undefined symbols have no address in this binary. + if (*FlagsOrErr & object::SymbolRef::SF_Undefined) + continue; + + Expected NameOrErr = Sym.getName(); + if (!NameOrErr) { + consumeError(NameOrErr.takeError()); + continue; + } + if (NameOrErr->empty()) + continue; + + Expected AddrOrErr = Sym.getValue(); + if (!AddrOrErr) { + consumeError(AddrOrErr.takeError()); + continue; + } + + MainBinarySymbolAddresses[*NameOrErr] = *AddrOrErr; + } +} + bool DwarfLinkerForBinary::link(const DebugMap &Map) { + buildMainBinarySymbolMap(Map); + if (Options.DWARFLinkerType == DsymutilDWARFLinkerType::Parallel) return linkImpl(Map, Options.FileType); @@ -958,6 +1005,16 @@ void DwarfLinkerForBinary::AddressManager::findValidRelocsMachO( if (const auto *Mapping = DMO.lookupSymbol(*SymbolName)) ValidRelocs.emplace_back(Offset64, RelocSize, Addend, Mapping->getKey(), Mapping->getValue()); + else if (auto GlobalIt = Linker.MainBinarySymbolAddresses.find(*SymbolName); + GlobalIt != Linker.MainBinarySymbolAddresses.end()) + // Cross-module reference: the target symbol is defined in another + // object and so is absent from this object's debug map. Resolve it via + // the linked binary's global symbol table instead. The addend (0 for a + // plain symbol reference) is preserved; [relocate] computes + // BinaryAddress + Addend. + ValidRelocs.emplace_back( + Offset64, RelocSize, Addend, *SymbolName, + SymbolMapping(std::nullopt, GlobalIt->getValue(), 0)); } else if (const auto *Mapping = DMO.lookupObjectAddress(SymAddress)) { // Do not store the addend. The addend was the address of the symbol in // the object file, the address in the binary that is stored in the debug @@ -1117,6 +1174,27 @@ DwarfLinkerForBinary::AddressManager::getExprOpAddressRelocAdjustment( return std::nullopt; } +std::optional +DwarfLinkerForBinary::AddressManager::getExprOpAddressLinkedAddress( + uint64_t ObjectAddress) { + if (!DbgMapObj) + return std::nullopt; + // The debug map records, for each symbol, its address in the object file and + // in the linked binary. A DW_OP_addr operand inside a location list holds the + // object-file address of such a symbol (e.g. a statically-allocated OCaml + // closure), so translate it to the corresponding linked address. + if (const DebugMapObject::DebugMapEntry *Mapping = + DbgMapObj->lookupObjectAddress(ObjectAddress)) { + uint64_t SymObjectAddress = + Mapping->getValue().ObjectAddress + ? uint64_t(*Mapping->getValue().ObjectAddress) + : ObjectAddress; + return uint64_t(Mapping->getValue().BinaryAddress) + + (ObjectAddress - SymObjectAddress); + } + return std::nullopt; +} + std::optional DwarfLinkerForBinary::AddressManager::getSubprogramRelocAdjustment( const DWARFDie &DIE, bool Verbose) { diff --git a/llvm/tools/dsymutil/DwarfLinkerForBinary.h b/llvm/tools/dsymutil/DwarfLinkerForBinary.h index 53f9e183ebe88..fc310b0e43cc5 100644 --- a/llvm/tools/dsymutil/DwarfLinkerForBinary.h +++ b/llvm/tools/dsymutil/DwarfLinkerForBinary.h @@ -121,6 +121,11 @@ class DwarfLinkerForBinary { std::optional LibInstallName; + /// The debug map object this manager was built from. Used to map the + /// object-file address of a DW_OP_addr operand inside a location list to + /// its linked address. Outlives this manager. + const DebugMapObject *DbgMapObj = nullptr; + /// Returns list of valid relocations from \p Relocs, /// between \p StartOffset and \p NextOffset. /// @@ -145,7 +150,8 @@ class DwarfLinkerForBinary { const DebugMapObject &DMO, std::shared_ptr DLBRM) : Linker(Linker), SrcFileName(DMO.getObjectFilename()), - DebugMapObjectType(MachO::N_OSO), DwarfLinkerRelocMap(DLBRM) { + DebugMapObjectType(MachO::N_OSO), DwarfLinkerRelocMap(DLBRM), + DbgMapObj(&DMO) { if (DMO.getRelocationMap().has_value()) { DebugMapObjectType = MachO::N_LIB; LibInstallName.emplace(DMO.getInstallName().value()); @@ -207,6 +213,9 @@ class DwarfLinkerForBinary { std::optional getSubprogramRelocAdjustment(const DWARFDie &DIE, bool Verbose) override; + std::optional + getExprOpAddressLinkedAddress(uint64_t ObjectAddress) override; + std::optional getLibraryInstallName() override; bool applyValidRelocs(MutableArrayRef Data, uint64_t BaseOffset, @@ -269,11 +278,22 @@ class DwarfLinkerForBinary { Error emitRelocations(const DebugMap &DM, std::vector &ObjectsForLinking); + /// Populate \p MainBinarySymbolAddresses from the linked binary's symbol + /// table, so that cross-module symbol references in debug info (whose target + /// symbol is absent from the referencing object's debug map) can still be + /// relocated. See \p AddressManager::findValidRelocsMachO. + void buildMainBinarySymbolMap(const DebugMap &Map); + raw_fd_ostream &OutFile; BinaryHolder &BinHolder; LinkOptions Options; std::mutex &ErrorHandlerMutex; + /// Maps a defined symbol name in the linked binary to its address. Used as a + /// fallback to resolve extern relocations against symbols defined in another + /// object (and hence missing from the current object's debug map). + StringMap MainBinarySymbolAddresses; + std::vector EmptyWarnings; /// A list of all .swiftinterface files referenced by the debug