Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion geonode/base/models.py
Original file line number Diff line number Diff line change
Expand Up @@ -971,7 +971,7 @@ def can_have_style(self):

@property
def can_have_thumbnail(self):
return self.subtype not in {"3dtiles", "cog"}
return self.subtype not in {"3dtiles", "cog", "flatgeobuf"}

@property
def raw_purpose(self):
Expand Down
9 changes: 5 additions & 4 deletions geonode/upload/handlers/common/remote.py
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,6 @@
from geonode.resource.enumerator import ExecutionRequestAction as exa
from geonode.upload.api.exceptions import ImportException
from geonode.upload.handlers.base import BaseHandler
from geonode.upload.handlers.common.serializer import RemoteResourceSerializer
from geonode.upload.models import ResourceHandlerInfo
from geonode.upload.orchestrator import orchestrator
from geonode.upload.celery_tasks import import_orchestrator
Expand Down Expand Up @@ -65,9 +64,11 @@ class BaseRemoteResourceHandler(BaseHandler):

@staticmethod
def has_serializer(data) -> bool:
if "url" in data:
return RemoteResourceSerializer
return False
"""
This method should be implemented by subclasses to determine if a serializer is available
for the given data.
"""
raise NotImplementedError("Subclasses must implement the 'has_serializer' method.")

@property
def have_table(self):
Expand Down
7 changes: 3 additions & 4 deletions geonode/upload/handlers/common/test_remote.py
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,6 @@
from geonode.upload.api.exceptions import ImportException
from geonode.upload.handlers.common.remote import BaseRemoteResourceHandler
from django.contrib.auth import get_user_model
from geonode.upload.handlers.common.serializer import RemoteResourceSerializer
from geonode.upload.orchestrator import orchestrator
from geonode.base.populate_test_data import create_single_dataset
from geonode.resource.models import ExecutionRequest
Expand Down Expand Up @@ -60,9 +59,9 @@ def test_can_handle_should_return_false_for_other_files(self):
actual = self.handler.can_handle({"base_file": "random.file"})
self.assertFalse(actual)

def test_should_get_the_specific_serializer(self):
actual = self.handler.has_serializer(self.valid_files)
self.assertEqual(type(actual), type(RemoteResourceSerializer))
def test_has_serializer_raises_not_implemented_error(self):
with self.assertRaises(NotImplementedError):
self.handler.has_serializer(self.valid_files)

def test_create_error_log(self):
"""
Expand Down
7 changes: 7 additions & 0 deletions geonode/upload/handlers/remote/cog.py
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@

from geonode.layers.models import Dataset
from geonode.upload.handlers.common.remote import BaseRemoteResourceHandler
from geonode.upload.handlers.common.serializer import RemoteResourceSerializer
from geonode.upload.api.exceptions import ImportException
from geonode.upload.orchestrator import orchestrator

Expand All @@ -34,6 +35,12 @@ class RemoteCOGResourceHandler(BaseRemoteResourceHandler):
def supported_file_extension_config(self):
return {}

@staticmethod
def has_serializer(data) -> bool:
if "url" in data and "cog" in data.get("type", "").lower():
return RemoteResourceSerializer
return False

@staticmethod
def can_handle(_data) -> bool:
"""
Expand Down
181 changes: 181 additions & 0 deletions geonode/upload/handlers/remote/flatgeobuf.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,181 @@
#########################################################################
#
# Copyright (C) 2026 OSGeo
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
#########################################################################
import logging
import requests
from osgeo import gdal

from geonode.layers.models import Dataset
from geonode.upload.handlers.common.remote import BaseRemoteResourceHandler
from geonode.upload.handlers.common.serializer import RemoteResourceSerializer
from geonode.upload.api.exceptions import ImportException
from geonode.upload.orchestrator import orchestrator
from geonode.geoserver.helpers import set_attributes

logger = logging.getLogger("importer")


class RemoteFlatGeobufResourceHandler(BaseRemoteResourceHandler):

@property
def supported_file_extension_config(self):
return {}

@staticmethod
def has_serializer(data) -> bool:
if "url" in data and "flatgeobuf" in data.get("type", "").lower():
return RemoteResourceSerializer
return False

@staticmethod
def can_handle(_data) -> bool:
"""
This endpoint will return True or False if with the info provided
the handler is able to handle the file or not
"""
if "url" in _data and "flatgeobuf" in _data.get("type", "").lower():
return True
return False

@staticmethod
def is_valid_url(url, **kwargs):
"""
Check if the URL is reachable and supports HTTP Range requests
"""
logger.debug(f"Checking FlatGeobuf URL validity (HEAD): {url}")
try:
# Reachability check using HEAD
head_res = requests.head(url, timeout=10, allow_redirects=True)
logger.debug(f"HTTP HEAD status: {head_res.status_code}")
head_res.raise_for_status()

accept_ranges = head_res.headers.get("Accept-Ranges", "").lower()

# Check for range request support
if accept_ranges == "bytes":
logger.debug("Server explicitly supports Accept-Ranges: bytes")
return True

# Some servers might not return Accept-Ranges in HEAD, so we try a small range request
logger.debug("Accept-Ranges header missing, trying a small Range GET...")
range_res = requests.get(url, headers={"Range": "bytes=0-1"}, timeout=10, stream=True)
logger.debug(f"Range GET status: {range_res.status_code}")
try:
if range_res.status_code != 206:
raise ImportException(
"The remote server does not support HTTP Range requests, which are required for FlatGeobuf."
)
finally:
range_res.close()
except ImportException as e:
raise e
except Exception as e:
logger.debug(f"is_valid_url ERROR: {str(e)}")
logger.exception(e)
raise ImportException("Error checking FlatGeobuf URL")

return True

def create_geonode_resource(
self,
layer_name: str,
alternate: str,
execution_id: str,
resource_type: Dataset = Dataset,
asset=None,
):
"""
Base function to create the resource into geonode.
Extracts metadata from remote FlatGeobuf using GDAL via HTTP range requests.
"""
logger.debug(f"Entering create_geonode_resource for {layer_name}")
_exec = orchestrator.get_execution_object(execution_id)
params = _exec.input_params.copy()
url = params.get("url")

# Extract metadata via GDAL VSICURL
gdal.UseExceptions()
logger.debug(f"Attempting to open FlatGeobuf with GDAL: /vsicurl/{url}")
try:
# Set GDAL config options for faster failure
gdal.SetThreadLocalConfigOption("GDAL_HTTP_TIMEOUT", "15")
gdal.SetThreadLocalConfigOption("GDAL_HTTP_MAX_RETRY", "1")

vsiurl = f"/vsicurl/{url}"
ds = gdal.OpenEx(
vsiurl,
allowed_drivers=["FlatGeobuf"],
)
if ds is None:
logger.debug(f"GDAL failed to open dataset: {vsiurl}")
raise ImportException(f"Could not open remote FlatGeobuf: {url}")

logger.debug("GDAL opened dataset. Extracting metadata...")

layer = ds.GetLayer(0)
if layer is None:
raise ImportException(f"No layers found in FlatGeobuf: {url}")

if not layer.GetSpatialRef():
raise ImportException(f"Could not extract spatial reference from Flatgeobuf: {url}")

srid = self.identify_authority(layer)

# Get BBox
try:
extent = layer.GetExtent()
bbox = [extent[0], extent[2], extent[1], extent[3]]
logger.debug(f"Extracted bounding box: {bbox}")
except Exception as e:
logger.error(f"Could not extract bounding box from FlatGeobuf: {url}. Error: {e}")
raise ImportException(
"Could not extract bounding box from FlatGeobuf. " "The file may be empty or corrupted."
)

# Get feature attributes
layer_defn = layer.GetLayerDefn()
attribute_map = []
for i in range(layer_defn.GetFieldCount()):
field_defn = layer_defn.GetFieldDefn(i)
attribute_map.append([field_defn.GetName(), field_defn.GetTypeName()])

logger.debug(f"Extracted schema with {len(attribute_map)} fields")
logger.debug("GDAL operations finished.")

ds = None # close dataset
except ImportException as e:
raise e
except Exception as e:
logger.debug(f"is_valid_url ERROR: {str(e)}")
logger.exception(e)
raise ImportException("Error checking FlatGeobuf URL")

resource = super().create_geonode_resource(layer_name, alternate, execution_id, resource_type, asset)
resource.set_bbox_polygon(bbox, srid)
set_attributes(resource, attribute_map)

return resource

def generate_resource_payload(self, layer_name, alternate, asset, _exec, workspace, **kwargs):
payload = super().generate_resource_payload(layer_name, alternate, asset, _exec, workspace, **kwargs)
payload.update(
{
"name": alternate,
}
)
return payload
123 changes: 123 additions & 0 deletions geonode/upload/handlers/remote/tests/test_flatgeobuf.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,123 @@
#########################################################################
#
# Copyright (C) 2026 OSGeo
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
#########################################################################
from django.test import TestCase
from mock import MagicMock, patch
from geonode.upload.api.exceptions import ImportException
from django.contrib.auth import get_user_model
from geonode.upload.handlers.remote.flatgeobuf import RemoteFlatGeobufResourceHandler
from geonode.upload.orchestrator import orchestrator
from geonode.layers.models import Dataset, Attribute
from geonode.base.models import Link


class TestRemoteFlatGeobufResourceHandler(TestCase):
databases = ("default", "datastore")

@classmethod
def setUpClass(cls):
super().setUpClass()
cls.handler = RemoteFlatGeobufResourceHandler()
cls.valid_url = "http://example.com/test.fgb"
cls.user, _ = get_user_model().objects.get_or_create(username="admin")
cls.valid_payload = {
"url": cls.valid_url,
"type": "flatgeobuf",
"title": "FlatGeobuf Test",
}
cls.owner = cls.user

def test_can_handle_flatgeobuf(self):
self.assertTrue(self.handler.can_handle(self.valid_payload))
self.assertTrue(self.handler.can_handle({"url": "http://example.com/y.fgb", "type": "flatgeobuf"}))
self.assertTrue(self.handler.can_handle({"url": "http://example.com/y.fgb", "type": "FlatGeobuf"}))
self.assertFalse(self.handler.can_handle({"url": "http://example.com/y.jpg", "type": "image"}))

@patch("geonode.upload.handlers.remote.flatgeobuf.requests.head")
@patch("geonode.upload.handlers.remote.flatgeobuf.requests.get")
@patch("geonode.upload.handlers.common.remote.requests.get")
def test_is_valid_url_success(self, mock_base_get, mock_get, mock_head):
mock_head.return_value.headers = {"Accept-Ranges": "bytes"}
mock_head.return_value.status_code = 200
mock_base_get.return_value.status_code = 200

self.assertTrue(self.handler.is_valid_url(self.valid_url))

@patch("geonode.upload.handlers.remote.flatgeobuf.requests.head")
@patch("geonode.upload.handlers.remote.flatgeobuf.requests.get")
@patch("geonode.upload.handlers.common.remote.requests.get")
def test_is_valid_url_no_range_support(self, mock_base_get, mock_get, mock_head):
mock_head.return_value.headers = {}
mock_get.return_value.status_code = 404
mock_base_get.return_value.status_code = 200

with self.assertRaises(ImportException):
self.handler.is_valid_url(self.valid_url)

@patch("geonode.upload.handlers.remote.flatgeobuf.gdal.OpenEx")
def test_create_geonode_resource(self, mock_gdal_openex):
# Mock GDAL dataset and layer
mock_ds = MagicMock()
mock_layer = MagicMock()
mock_srs = MagicMock()
mock_layer_defn = MagicMock()
mock_field_defn = MagicMock()

mock_srs.GetAuthorityName.return_value = "EPSG"
mock_srs.GetAuthorityCode.return_value = "4326"
mock_layer.GetSpatialRef.return_value = mock_srs

mock_layer.GetExtent.return_value = [-180.0, 180.0, -90.0, 90.0]

mock_field_defn.GetName.return_value = "test_field"
mock_field_defn.GetTypeName.return_value = "String"
mock_layer_defn.GetFieldCount.return_value = 1
mock_layer_defn.GetFieldDefn.return_value = mock_field_defn
mock_layer.GetLayerDefn.return_value = mock_layer_defn

mock_ds.GetLayer.return_value = mock_layer
mock_gdal_openex.return_value = mock_ds

exec_id = orchestrator.create_execution_request(
user=self.owner,
func_name="funct1",
step="step",
input_params=self.valid_payload,
)

resource = self.handler.create_geonode_resource(
"test_flatgeobuf",
"test_flatgeobuf_alternate",
execution_id=str(exec_id),
resource_type=Dataset,
)

self.assertIsNotNone(resource)
self.assertEqual(resource.subtype, "flatgeobuf")
self.assertEqual(resource.alternate, "test_flatgeobuf_alternate")
self.assertEqual(resource.srid, "EPSG:4326")
self.assertIsNotNone(resource.bbox_polygon)

link = Link.objects.get(resource=resource, link_type="data")
self.assertEqual(link.url, self.valid_url)
self.assertEqual(link.extension, "flatgeobuf")
self.assertEqual(link.name, resource.alternate)

# Verify Attribute metadata persistence
attrs = Attribute.objects.filter(dataset=resource)
self.assertTrue(attrs.filter(attribute="test_field").exists())
1 change: 1 addition & 0 deletions geonode/upload/settings.py
Original file line number Diff line number Diff line change
Expand Up @@ -37,6 +37,7 @@
"geonode.upload.handlers.sld.handler.SLDFileHandler",
"geonode.upload.handlers.tiles3d.handler.Tiles3DFileHandler",
"geonode.upload.handlers.remote.tiles3d.RemoteTiles3DResourceHandler",
"geonode.upload.handlers.remote.flatgeobuf.RemoteFlatGeobufResourceHandler",
"geonode.upload.handlers.remote.wms.RemoteWMSResourceHandler",
"geonode.upload.handlers.remote.cog.RemoteCOGResourceHandler",
"geonode.upload.handlers.empty_dataset.handler.EmptyDatasetHandler",
Expand Down
Loading