Updated field annotations for queryables, roles, and json_path (#61)

Author: lukasbindreiter

apis/datasets/v1/data_access.proto

@@ -7,6 +7,8 @@ package datasets.v1;
 import "buf/validate/validate.proto";
 import "datasets/v1/core.proto";
 import "datasets/v1/well_known_types.proto";
+import "google/protobuf/duration.proto";
+import "google/protobuf/timestamp.proto";
 import "tilebox/v1/id.proto";
 import "tilebox/v1/query.proto";
 
@@ -41,6 +43,125 @@ message QueryFilters {
   tilebox.v1.IDInterval datapoint_interval = 2;
 
   SpatialFilter spatial_extent = 3;
+
+  // Additional expressions over fields marked queryable in the dataset schema.
+  // All top-level expressions are combined with each other and with the interval and spatial filters using logical AND.
+  // Use a nested LogicalExpression for explicit OR, NOT, or grouped boolean logic.
+  repeated FilterExpression expressions = 4 [(buf.validate.field).repeated.max_items = 100];
+}
+
+// FilterExpression is a typed filter expression compatible with the Basic CQL2 property-to-literal subset.
+message FilterExpression {
+  option (buf.validate.message).oneof = {
+    fields: [
+      "logical",
+      "comparison",
+      "is_null"
+    ]
+    required: true
+  };
+
+  // Exactly one expression node must be set.
+  LogicalExpression logical = 1;
+  FieldComparison comparison = 2;
+  FieldNullCheck is_null = 3;
+}
+
+// LogicalExpression combines nested filter expressions.
+message LogicalExpression {
+  option (buf.validate.message).cel = {
+    id: "logical_expression.operand_arity"
+    message: "NOT requires exactly one operand; AND and OR require at least two"
+    expression: "this.operator == 3 ? this.operands.size() == 1 : this.operands.size() >= 2"
+  };
+
+  // The logical operation to apply to operands.
+  LogicalOperator operator = 1 [(buf.validate.field).enum = {
+    defined_only: true
+    not_in: [0]
+  }];
+  // Operands for the logical operation. AND and OR require at least two; NOT requires exactly one.
+  repeated FilterExpression operands = 2 [(buf.validate.field).repeated = {
+    min_items: 1
+    max_items: 100
+  }];
+}
+
+// LogicalOperator specifies how nested filter expressions are combined.
+enum LogicalOperator {
+  LOGICAL_OPERATOR_UNSPECIFIED = 0;
+  LOGICAL_OPERATOR_AND = 1;
+  LOGICAL_OPERATOR_OR = 2;
+  LOGICAL_OPERATOR_NOT = 3;
+}
+
+// FieldComparison compares a queryable dataset field to a typed literal value.
+message FieldComparison {
+  // Name of a top-level field in the dataset schema.
+  string field_name = 1 [(buf.validate.field).string = {
+    min_len: 1
+    max_len: 100
+    pattern: "^[a-z][a-z0-9_]*$"
+  }];
+  // Comparison operation to apply.
+  FieldComparisonOperator operator = 2 [(buf.validate.field).enum = {
+    defined_only: true
+    not_in: [0]
+  }];
+  // Literal value. Its kind must match the field's protobuf descriptor.
+  FieldQueryValue value = 3 [(buf.validate.field).required = true];
+}
+
+// FieldComparisonOperator specifies a Basic CQL2 comparison operation.
+enum FieldComparisonOperator {
+  FIELD_COMPARISON_OPERATOR_UNSPECIFIED = 0;
+  FIELD_COMPARISON_OPERATOR_EQUAL = 1;
+  FIELD_COMPARISON_OPERATOR_NOT_EQUAL = 2;
+  FIELD_COMPARISON_OPERATOR_LESS_THAN = 3;
+  FIELD_COMPARISON_OPERATOR_LESS_THAN_OR_EQUAL = 4;
+  FIELD_COMPARISON_OPERATOR_GREATER_THAN = 5;
+  FIELD_COMPARISON_OPERATOR_GREATER_THAN_OR_EQUAL = 6;
+}
+
+// FieldNullCheck matches datapoints for which a queryable field is absent or explicitly null.
+// IS NOT NULL is represented by wrapping this expression in LOGICAL_OPERATOR_NOT.
+message FieldNullCheck {
+  // Name of a top-level field in the dataset schema.
+  string field_name = 1 [(buf.validate.field).string = {
+    min_len: 1
+    max_len: 100
+    pattern: "^[a-z][a-z0-9_]*$"
+  }];
+}
+
+// FieldQueryValue is a typed scalar or well-known-type literal used in a field comparison.
+message FieldQueryValue {
+  option (buf.validate.message).oneof = {
+    fields: [
+      "bool_value",
+      "int64_value",
+      "uint64_value",
+      "double_value",
+      "string_value",
+      "timestamp_value",
+      "duration_value",
+      "enum_name",
+      "bytes_value"
+    ]
+    required: true
+  };
+
+  // Exactly one typed literal must be set. Explicit presence preserves zero and empty values.
+  bool bool_value = 1 [features.field_presence = EXPLICIT];
+  int64 int64_value = 2 [features.field_presence = EXPLICIT];
+  uint64 uint64_value = 3 [features.field_presence = EXPLICIT];
+  double double_value = 4 [features.field_presence = EXPLICIT];
+  string string_value = 5 [features.field_presence = EXPLICIT];
+  google.protobuf.Timestamp timestamp_value = 6;
+  google.protobuf.Duration duration_value = 7;
+  // Symbolic value from the field's enum descriptor.
+  string enum_name = 8 [features.field_presence = EXPLICIT];
+  bytes bytes_value = 9 [features.field_presence = EXPLICIT];
 }
 
 // SpatialFilterMode specifies how geometries are compared to a given spatial filter.

apis/datasets/v1/dataset_type.proto

@@ -12,19 +12,19 @@ import "tilebox/v1/id.proto";
 
 option features.field_presence = IMPLICIT;
 
-// Field describes a field of a dataset.
+// Field is the request representation of a dataset field. It intentionally combines only a protobuf field descriptor
+// with its Tilebox annotation, matching the information represented separately by AnnotatedType.
 message Field {
+  reserved 3;
+  reserved queryable;
+
   // The descriptor contains the name of the field, the type, optional labels (e.g. repeated) and other information.
   // If the type is TYPE_MESSAGE, then the type_name must be a fully qualified name to a well known type, e.g.
   // `datasets.v1.Vec3` or `google.protobuf.Timestamp`.
   google.protobuf.FieldDescriptorProto descriptor = 1 [(buf.validate.field).required = true];
 
-  // An optional description and example value for the field.
+  // Additional documentation, source mapping, and query metadata for the field.
   FieldAnnotation annotation = 2;
-
-  // A flag indicating whether the field should be queryable. This means we will build an index for the field, and
-  // allow users to query for certain values server-side.
-  bool queryable = 3 [(buf.validate.field).bool.const = false];
 }
 
 // DatasetKind is an enum describing the kind of dataset. A dataset kind specifies a set of default fields, that
@@ -40,13 +40,44 @@ enum DatasetKind {
   DATASET_KIND_SPATIOTEMPORAL = 2;
 }
 
-// FieldAnnotation contains additional information about a field in a dataset
+// FieldRole describes a semantic display role fulfilled by a dataset field.
+// Thumbnail discovery is intentionally not represented here: thumbnails are assets whose STAC asset roles include
+// thumbnail.
+enum FieldRole {
+  FIELD_ROLE_UNSPECIFIED = 0;
+
+  // The primary human-readable title or name used when displaying a datapoint.
+  FIELD_ROLE_PRIMARY_TITLE = 1;
+}
+
+// FieldAnnotation contains additional information about a field in a dataset.
 message FieldAnnotation {
   string description = 1;
   string example_value = 2;
+
+  // RFC 6901 JSON Pointer locating the field value in the source document.
+  // This allows flattened dataset fields to be reconstructed without relying on their protobuf field names.
+  string source_json_pointer = 3 [features.field_presence = EXPLICIT];
+
+  // Whether the field is projected into query storage and may be used in server-side filter expressions.
+  // Queryable fields are not necessarily backed by a secondary database index.
+  bool queryable = 4;
+
+  // Optional JSON Schema reference to emit as `$ref` when advertising this field as a queryable.
+  // For example, this may reference the canonical STAC extension schema definition for a property.
+  string queryable_json_schema_ref = 5 [features.field_presence = EXPLICIT];
+
+  // Semantic display roles fulfilled by this field.
+  repeated FieldRole roles = 6 [
+    (buf.validate.field).repeated.unique = true,
+    (buf.validate.field).repeated.items.enum = {
+      defined_only: true
+      not_in: [0]
+    }
+  ];
 }
 
-// DatasetType describes the type of a dataset.
+// DatasetType is the request representation of a dataset type, used when creating or updating a dataset.
 message DatasetType {
   // kind denotes the kind of dataset this type describes. We do not rely on the default fields to be set in our
   // array of fields - since that way users could circumvent those by manually sending requests not from our Console.
@@ -59,12 +90,15 @@ message DatasetType {
   repeated Field fields = 2;
 }
 
-// AnnotatedType describes a message type
+// AnnotatedType is the resolved representation returned on Dataset messages, including GetDataset responses.
+// It combines the generated protobuf descriptors with the corresponding Tilebox field annotations.
 message AnnotatedType {
   google.protobuf.FileDescriptorSet descriptor_set = 1;
   // the url of the type, one of the types defined in the descriptor
   string type_url = 2;
   reserved 3;
+  // Annotations corresponding by index to the FieldDescriptorProto entries of the message selected by type_url.
+  // Empty annotations must be retained so this positional mapping remains stable.
   repeated FieldAnnotation field_annotations = 4;
   DatasetKind kind = 5;
 }

apis/datasets/v1/datasets.proto

@@ -19,7 +19,7 @@ message CreateDatasetRequest {
     (buf.validate.field).string.min_len = 1,
     (buf.validate.field).string.max_len = 100
   ];
-  // message type of the dataset to create.
+  // Message type of the dataset to create, including per-field source mappings, queryable metadata, and roles.
   DatasetType type = 2 [(buf.validate.field).required = true];
   // short text summary of the dataset to create.
   string summary = 3 [(buf.validate.field).string.max_len = 50000];
@@ -62,7 +62,7 @@ message UpdateDatasetRequest {
     (buf.validate.field).string.min_len = 1,
     (buf.validate.field).string.max_len = 100
   ];
-  // updated type of the dataset.
+  // Updated type of the dataset, including per-field source mappings, queryable metadata, and roles.
   DatasetType type = 3 [features.field_presence = EXPLICIT];
   // updated summary of the dataset.
   string summary = 4 [