diff --git a/geonode/base/models.py b/geonode/base/models.py index eeb39502d35..6ead4558734 100644 --- a/geonode/base/models.py +++ b/geonode/base/models.py @@ -971,7 +971,7 @@ def can_have_style(self): @property def can_have_thumbnail(self): - return self.subtype not in {"3dtiles", "cog"} + return self.subtype not in {"3dtiles", "cog", "flatgeobuf"} @property def raw_purpose(self): diff --git a/geonode/upload/handlers/common/remote.py b/geonode/upload/handlers/common/remote.py index 36699f20bb1..97113fbc6dc 100755 --- a/geonode/upload/handlers/common/remote.py +++ b/geonode/upload/handlers/common/remote.py @@ -25,7 +25,6 @@ from geonode.resource.enumerator import ExecutionRequestAction as exa from geonode.upload.api.exceptions import ImportException from geonode.upload.handlers.base import BaseHandler -from geonode.upload.handlers.common.serializer import RemoteResourceSerializer from geonode.upload.models import ResourceHandlerInfo from geonode.upload.orchestrator import orchestrator from geonode.upload.celery_tasks import import_orchestrator @@ -65,9 +64,11 @@ class BaseRemoteResourceHandler(BaseHandler): @staticmethod def has_serializer(data) -> bool: - if "url" in data: - return RemoteResourceSerializer - return False + """ + This method should be implemented by subclasses to determine if a serializer is available + for the given data. + """ + raise NotImplementedError("Subclasses must implement the 'has_serializer' method.") @property def have_table(self): diff --git a/geonode/upload/handlers/common/test_remote.py b/geonode/upload/handlers/common/test_remote.py index c61b02981f1..5c78c138cbe 100644 --- a/geonode/upload/handlers/common/test_remote.py +++ b/geonode/upload/handlers/common/test_remote.py @@ -21,7 +21,6 @@ from geonode.upload.api.exceptions import ImportException from geonode.upload.handlers.common.remote import BaseRemoteResourceHandler from django.contrib.auth import get_user_model -from geonode.upload.handlers.common.serializer import RemoteResourceSerializer from geonode.upload.orchestrator import orchestrator from geonode.base.populate_test_data import create_single_dataset from geonode.resource.models import ExecutionRequest @@ -60,9 +59,9 @@ def test_can_handle_should_return_false_for_other_files(self): actual = self.handler.can_handle({"base_file": "random.file"}) self.assertFalse(actual) - def test_should_get_the_specific_serializer(self): - actual = self.handler.has_serializer(self.valid_files) - self.assertEqual(type(actual), type(RemoteResourceSerializer)) + def test_has_serializer_raises_not_implemented_error(self): + with self.assertRaises(NotImplementedError): + self.handler.has_serializer(self.valid_files) def test_create_error_log(self): """ diff --git a/geonode/upload/handlers/remote/cog.py b/geonode/upload/handlers/remote/cog.py index 27b946457b5..75e958b80f4 100644 --- a/geonode/upload/handlers/remote/cog.py +++ b/geonode/upload/handlers/remote/cog.py @@ -22,6 +22,7 @@ from geonode.layers.models import Dataset from geonode.upload.handlers.common.remote import BaseRemoteResourceHandler +from geonode.upload.handlers.common.serializer import RemoteResourceSerializer from geonode.upload.api.exceptions import ImportException from geonode.upload.orchestrator import orchestrator @@ -34,6 +35,12 @@ class RemoteCOGResourceHandler(BaseRemoteResourceHandler): def supported_file_extension_config(self): return {} + @staticmethod + def has_serializer(data) -> bool: + if "url" in data and "cog" in data.get("type", "").lower(): + return RemoteResourceSerializer + return False + @staticmethod def can_handle(_data) -> bool: """ diff --git a/geonode/upload/handlers/remote/flatgeobuf.py b/geonode/upload/handlers/remote/flatgeobuf.py new file mode 100644 index 00000000000..83dcb26e4bb --- /dev/null +++ b/geonode/upload/handlers/remote/flatgeobuf.py @@ -0,0 +1,181 @@ +######################################################################### +# +# Copyright (C) 2026 OSGeo +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +######################################################################### +import logging +import requests +from osgeo import gdal + +from geonode.layers.models import Dataset +from geonode.upload.handlers.common.remote import BaseRemoteResourceHandler +from geonode.upload.handlers.common.serializer import RemoteResourceSerializer +from geonode.upload.api.exceptions import ImportException +from geonode.upload.orchestrator import orchestrator +from geonode.geoserver.helpers import set_attributes + +logger = logging.getLogger("importer") + + +class RemoteFlatGeobufResourceHandler(BaseRemoteResourceHandler): + + @property + def supported_file_extension_config(self): + return {} + + @staticmethod + def has_serializer(data) -> bool: + if "url" in data and "flatgeobuf" in data.get("type", "").lower(): + return RemoteResourceSerializer + return False + + @staticmethod + def can_handle(_data) -> bool: + """ + This endpoint will return True or False if with the info provided + the handler is able to handle the file or not + """ + if "url" in _data and "flatgeobuf" in _data.get("type", "").lower(): + return True + return False + + @staticmethod + def is_valid_url(url, **kwargs): + """ + Check if the URL is reachable and supports HTTP Range requests + """ + logger.debug(f"Checking FlatGeobuf URL validity (HEAD): {url}") + try: + # Reachability check using HEAD + head_res = requests.head(url, timeout=10, allow_redirects=True) + logger.debug(f"HTTP HEAD status: {head_res.status_code}") + head_res.raise_for_status() + + accept_ranges = head_res.headers.get("Accept-Ranges", "").lower() + + # Check for range request support + if accept_ranges == "bytes": + logger.debug("Server explicitly supports Accept-Ranges: bytes") + return True + + # Some servers might not return Accept-Ranges in HEAD, so we try a small range request + logger.debug("Accept-Ranges header missing, trying a small Range GET...") + range_res = requests.get(url, headers={"Range": "bytes=0-1"}, timeout=10, stream=True) + logger.debug(f"Range GET status: {range_res.status_code}") + try: + if range_res.status_code != 206: + raise ImportException( + "The remote server does not support HTTP Range requests, which are required for FlatGeobuf." + ) + finally: + range_res.close() + except ImportException as e: + raise e + except Exception as e: + logger.debug(f"is_valid_url ERROR: {str(e)}") + logger.exception(e) + raise ImportException("Error checking FlatGeobuf URL") + + return True + + def create_geonode_resource( + self, + layer_name: str, + alternate: str, + execution_id: str, + resource_type: Dataset = Dataset, + asset=None, + ): + """ + Base function to create the resource into geonode. + Extracts metadata from remote FlatGeobuf using GDAL via HTTP range requests. + """ + logger.debug(f"Entering create_geonode_resource for {layer_name}") + _exec = orchestrator.get_execution_object(execution_id) + params = _exec.input_params.copy() + url = params.get("url") + + # Extract metadata via GDAL VSICURL + gdal.UseExceptions() + logger.debug(f"Attempting to open FlatGeobuf with GDAL: /vsicurl/{url}") + try: + # Set GDAL config options for faster failure + gdal.SetThreadLocalConfigOption("GDAL_HTTP_TIMEOUT", "15") + gdal.SetThreadLocalConfigOption("GDAL_HTTP_MAX_RETRY", "1") + + vsiurl = f"/vsicurl/{url}" + ds = gdal.OpenEx( + vsiurl, + allowed_drivers=["FlatGeobuf"], + ) + if ds is None: + logger.debug(f"GDAL failed to open dataset: {vsiurl}") + raise ImportException(f"Could not open remote FlatGeobuf: {url}") + + logger.debug("GDAL opened dataset. Extracting metadata...") + + layer = ds.GetLayer(0) + if layer is None: + raise ImportException(f"No layers found in FlatGeobuf: {url}") + + if not layer.GetSpatialRef(): + raise ImportException(f"Could not extract spatial reference from Flatgeobuf: {url}") + + srid = self.identify_authority(layer) + + # Get BBox + try: + extent = layer.GetExtent() + bbox = [extent[0], extent[2], extent[1], extent[3]] + logger.debug(f"Extracted bounding box: {bbox}") + except Exception as e: + logger.error(f"Could not extract bounding box from FlatGeobuf: {url}. Error: {e}") + raise ImportException( + "Could not extract bounding box from FlatGeobuf. " "The file may be empty or corrupted." + ) + + # Get feature attributes + layer_defn = layer.GetLayerDefn() + attribute_map = [] + for i in range(layer_defn.GetFieldCount()): + field_defn = layer_defn.GetFieldDefn(i) + attribute_map.append([field_defn.GetName(), field_defn.GetTypeName()]) + + logger.debug(f"Extracted schema with {len(attribute_map)} fields") + logger.debug("GDAL operations finished.") + + ds = None # close dataset + except ImportException as e: + raise e + except Exception as e: + logger.debug(f"is_valid_url ERROR: {str(e)}") + logger.exception(e) + raise ImportException("Error checking FlatGeobuf URL") + + resource = super().create_geonode_resource(layer_name, alternate, execution_id, resource_type, asset) + resource.set_bbox_polygon(bbox, srid) + set_attributes(resource, attribute_map) + + return resource + + def generate_resource_payload(self, layer_name, alternate, asset, _exec, workspace, **kwargs): + payload = super().generate_resource_payload(layer_name, alternate, asset, _exec, workspace, **kwargs) + payload.update( + { + "name": alternate, + } + ) + return payload diff --git a/geonode/upload/handlers/remote/tests/test_flatgeobuf.py b/geonode/upload/handlers/remote/tests/test_flatgeobuf.py new file mode 100644 index 00000000000..0776f0fa368 --- /dev/null +++ b/geonode/upload/handlers/remote/tests/test_flatgeobuf.py @@ -0,0 +1,123 @@ +######################################################################### +# +# Copyright (C) 2026 OSGeo +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +######################################################################### +from django.test import TestCase +from mock import MagicMock, patch +from geonode.upload.api.exceptions import ImportException +from django.contrib.auth import get_user_model +from geonode.upload.handlers.remote.flatgeobuf import RemoteFlatGeobufResourceHandler +from geonode.upload.orchestrator import orchestrator +from geonode.layers.models import Dataset, Attribute +from geonode.base.models import Link + + +class TestRemoteFlatGeobufResourceHandler(TestCase): + databases = ("default", "datastore") + + @classmethod + def setUpClass(cls): + super().setUpClass() + cls.handler = RemoteFlatGeobufResourceHandler() + cls.valid_url = "http://example.com/test.fgb" + cls.user, _ = get_user_model().objects.get_or_create(username="admin") + cls.valid_payload = { + "url": cls.valid_url, + "type": "flatgeobuf", + "title": "FlatGeobuf Test", + } + cls.owner = cls.user + + def test_can_handle_flatgeobuf(self): + self.assertTrue(self.handler.can_handle(self.valid_payload)) + self.assertTrue(self.handler.can_handle({"url": "http://example.com/y.fgb", "type": "flatgeobuf"})) + self.assertTrue(self.handler.can_handle({"url": "http://example.com/y.fgb", "type": "FlatGeobuf"})) + self.assertFalse(self.handler.can_handle({"url": "http://example.com/y.jpg", "type": "image"})) + + @patch("geonode.upload.handlers.remote.flatgeobuf.requests.head") + @patch("geonode.upload.handlers.remote.flatgeobuf.requests.get") + @patch("geonode.upload.handlers.common.remote.requests.get") + def test_is_valid_url_success(self, mock_base_get, mock_get, mock_head): + mock_head.return_value.headers = {"Accept-Ranges": "bytes"} + mock_head.return_value.status_code = 200 + mock_base_get.return_value.status_code = 200 + + self.assertTrue(self.handler.is_valid_url(self.valid_url)) + + @patch("geonode.upload.handlers.remote.flatgeobuf.requests.head") + @patch("geonode.upload.handlers.remote.flatgeobuf.requests.get") + @patch("geonode.upload.handlers.common.remote.requests.get") + def test_is_valid_url_no_range_support(self, mock_base_get, mock_get, mock_head): + mock_head.return_value.headers = {} + mock_get.return_value.status_code = 404 + mock_base_get.return_value.status_code = 200 + + with self.assertRaises(ImportException): + self.handler.is_valid_url(self.valid_url) + + @patch("geonode.upload.handlers.remote.flatgeobuf.gdal.OpenEx") + def test_create_geonode_resource(self, mock_gdal_openex): + # Mock GDAL dataset and layer + mock_ds = MagicMock() + mock_layer = MagicMock() + mock_srs = MagicMock() + mock_layer_defn = MagicMock() + mock_field_defn = MagicMock() + + mock_srs.GetAuthorityName.return_value = "EPSG" + mock_srs.GetAuthorityCode.return_value = "4326" + mock_layer.GetSpatialRef.return_value = mock_srs + + mock_layer.GetExtent.return_value = [-180.0, 180.0, -90.0, 90.0] + + mock_field_defn.GetName.return_value = "test_field" + mock_field_defn.GetTypeName.return_value = "String" + mock_layer_defn.GetFieldCount.return_value = 1 + mock_layer_defn.GetFieldDefn.return_value = mock_field_defn + mock_layer.GetLayerDefn.return_value = mock_layer_defn + + mock_ds.GetLayer.return_value = mock_layer + mock_gdal_openex.return_value = mock_ds + + exec_id = orchestrator.create_execution_request( + user=self.owner, + func_name="funct1", + step="step", + input_params=self.valid_payload, + ) + + resource = self.handler.create_geonode_resource( + "test_flatgeobuf", + "test_flatgeobuf_alternate", + execution_id=str(exec_id), + resource_type=Dataset, + ) + + self.assertIsNotNone(resource) + self.assertEqual(resource.subtype, "flatgeobuf") + self.assertEqual(resource.alternate, "test_flatgeobuf_alternate") + self.assertEqual(resource.srid, "EPSG:4326") + self.assertIsNotNone(resource.bbox_polygon) + + link = Link.objects.get(resource=resource, link_type="data") + self.assertEqual(link.url, self.valid_url) + self.assertEqual(link.extension, "flatgeobuf") + self.assertEqual(link.name, resource.alternate) + + # Verify Attribute metadata persistence + attrs = Attribute.objects.filter(dataset=resource) + self.assertTrue(attrs.filter(attribute="test_field").exists()) diff --git a/geonode/upload/settings.py b/geonode/upload/settings.py index e021a030395..4ac8621c2e5 100644 --- a/geonode/upload/settings.py +++ b/geonode/upload/settings.py @@ -37,6 +37,7 @@ "geonode.upload.handlers.sld.handler.SLDFileHandler", "geonode.upload.handlers.tiles3d.handler.Tiles3DFileHandler", "geonode.upload.handlers.remote.tiles3d.RemoteTiles3DResourceHandler", + "geonode.upload.handlers.remote.flatgeobuf.RemoteFlatGeobufResourceHandler", "geonode.upload.handlers.remote.wms.RemoteWMSResourceHandler", "geonode.upload.handlers.remote.cog.RemoteCOGResourceHandler", "geonode.upload.handlers.empty_dataset.handler.EmptyDatasetHandler",