From 2896d55651c882bbd322e57068ca6d4577915a6a Mon Sep 17 00:00:00 2001 From: Yufeng Xin Date: Tue, 7 Oct 2025 12:37:03 -0400 Subject: [PATCH 1/9] call dynamic client package --- env.template | 3 +++ sdx_lc/app.py | 3 ++- sdx_lc/jobs/pull_topo_changes.py | 20 +++++++++++++++++--- 3 files changed, 22 insertions(+), 4 deletions(-) diff --git a/env.template b/env.template index 7da7d88..cc130b7 100644 --- a/env.template +++ b/env.template @@ -38,3 +38,6 @@ OXP_PROVISION_URL=http://192.168.201.205:8088/SDX-LC/1.0.0/provision OXP_PULL_URL=http://192.168.201.205:8088/SDX-LC/1.0.0/topology OXP_PULL_INTERVAL=180 OXP_CONNECTION_URL=http://192.168.201.205:8088/SDX-LC/1.0.0/connection + +#OXP Client +CLIENT_PACKAGE: sense_sdx \ No newline at end of file diff --git a/sdx_lc/app.py b/sdx_lc/app.py index 5223a14..93d0a01 100644 --- a/sdx_lc/app.py +++ b/sdx_lc/app.py @@ -57,7 +57,8 @@ def create_app(): logger.info( f"SDX Local Controller starting up (" f"name: {os.getenv('SDXLC_NAME')}, " - f"domain: {os.getenv('SDXLC_DOMAIN')})" + f"domain: {os.getenv('SDXLC_DOMAIN')})," + f"client package:{os.getenv('CLIENT_PACKAGE')}" ) # Run swagger service diff --git a/sdx_lc/jobs/pull_topo_changes.py b/sdx_lc/jobs/pull_topo_changes.py index 7adce92..a8a214a 100644 --- a/sdx_lc/jobs/pull_topo_changes.py +++ b/sdx_lc/jobs/pull_topo_changes.py @@ -1,3 +1,4 @@ +import importlib import json import logging import os.path @@ -15,6 +16,9 @@ from messaging.rpc_queue_producer import RpcProducer from utils.db_utils import DbUtils +CLIENT_PACKAGE = os.getenv("CLIENT_PACKAGE", None) +CLIENT_TOPOLOGY_FUNCTION = os.getenv("CLIENT_TOPOLOGY_FUNCTION", None) + OXPO_USER = os.environ.get("OXPO_USER", None) OXPO_PASS = os.environ.get("OXPO_PASS", None) OXP_PULL_URL = os.environ.get("OXP_PULL_URL") @@ -47,7 +51,12 @@ def process_domain_controller_topo(db_instance): continue try: - latest_topo_version = json_latest_topology["version"] + if not CLIENT_PACKAGE: + latest_topo_version = json_latest_topology["version"] + else: + client_module = importlib.import_module(CLIENT_PACKAGE) + topology_function = getattr(client_module, CLIENT_TOPOLOGY_FUNCTION) + topology_function() except KeyError: logger.debug("Error getting topo version") continue @@ -55,8 +64,13 @@ def process_domain_controller_topo(db_instance): logger.debug("Latest topology does not exist") try: - response = requests.get(OXP_PULL_URL, auth=(OXPO_USER, OXPO_PASS)) - pulled_topology = response.content + if not CLIENT_PACKAGE: + response = requests.get(OXP_PULL_URL, auth=(OXPO_USER, OXPO_PASS)) + pulled_topology = response.content + else: + client_module = importlib.import_module(CLIENT_PACKAGE) + topology_function = getattr(client_module, CLIENT_TOPOLOGY_FUNCTION) + topology_function() except (requests.ConnectionError, requests.HTTPError): logger.debug("Error connecting to domain controller...") continue From c71022f8199708dc367702a10b7abd160cf67ea7 Mon Sep 17 00:00:00 2001 From: Yufeng Xin Date: Tue, 21 Oct 2025 22:43:09 -0400 Subject: [PATCH 2/9] cleanup --- env.template | 3 +- pyproject.toml | 1 + sdx_lc/app.py | 3 ++ sdx_lc/jobs/pull_topo_changes.py | 60 +++++++++++++------------------- 4 files changed, 30 insertions(+), 37 deletions(-) diff --git a/env.template b/env.template index cc130b7..27b25e6 100644 --- a/env.template +++ b/env.template @@ -40,4 +40,5 @@ OXP_PULL_INTERVAL=180 OXP_CONNECTION_URL=http://192.168.201.205:8088/SDX-LC/1.0.0/connection #OXP Client -CLIENT_PACKAGE: sense_sdx \ No newline at end of file +CLIENT_PACKAGE: sense_sdx +CLIENT_FUNCTION: client \ No newline at end of file diff --git a/pyproject.toml b/pyproject.toml index 13d7cb7..7dbb04a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -27,6 +27,7 @@ dependencies = [ "connexion[swagger-ui] == 2.14.2", "asgiref >= 3.7.2", "pymongo > 3.0", + "dotenv", "sdx-datamodel @ git+https://github.com/atlanticwave-sdx/datamodel@3.0.0.dev5", ] diff --git a/sdx_lc/app.py b/sdx_lc/app.py index 93d0a01..5b80de6 100644 --- a/sdx_lc/app.py +++ b/sdx_lc/app.py @@ -1,6 +1,7 @@ #!/usr/bin/env python3 import logging +from dotenv import load_dotenv import os import threading from queue import Queue @@ -43,6 +44,8 @@ def create_app(): Create a Flas/Connexion App. """ + load_dotenv() + logger = logging.getLogger(__name__) logging.getLogger("pika").setLevel(logging.WARNING) diff --git a/sdx_lc/jobs/pull_topo_changes.py b/sdx_lc/jobs/pull_topo_changes.py index a8a214a..e8cc957 100644 --- a/sdx_lc/jobs/pull_topo_changes.py +++ b/sdx_lc/jobs/pull_topo_changes.py @@ -17,7 +17,7 @@ from utils.db_utils import DbUtils CLIENT_PACKAGE = os.getenv("CLIENT_PACKAGE", None) -CLIENT_TOPOLOGY_FUNCTION = os.getenv("CLIENT_TOPOLOGY_FUNCTION", None) +CLIENT_FUNCTION = os.getenv("CLIENT_FUNCTION", None) OXPO_USER = os.environ.get("OXPO_USER", None) OXPO_PASS = os.environ.get("OXPO_PASS", None) @@ -51,49 +51,37 @@ def process_domain_controller_topo(db_instance): continue try: - if not CLIENT_PACKAGE: - latest_topo_version = json_latest_topology["version"] - else: - client_module = importlib.import_module(CLIENT_PACKAGE) - topology_function = getattr(client_module, CLIENT_TOPOLOGY_FUNCTION) - topology_function() + latest_topo_version = json_latest_topology["version"] except KeyError: logger.debug("Error getting topo version") continue else: logger.debug("Latest topology does not exist") - try: - if not CLIENT_PACKAGE: + if not CLIENT_PACKAGE: + try: response = requests.get(OXP_PULL_URL, auth=(OXPO_USER, OXPO_PASS)) pulled_topology = response.content - else: - client_module = importlib.import_module(CLIENT_PACKAGE) - topology_function = getattr(client_module, CLIENT_TOPOLOGY_FUNCTION) - topology_function() - except (requests.ConnectionError, requests.HTTPError): - logger.debug("Error connecting to domain controller...") - continue - - if not response.ok: - continue - - logger.debug("Pulled request from domain controller") - - try: - json_pulled_topology = response.json() - except ValueError: - logger.debug("Cannot parse pulled topology, invalid JSON") - continue - - try: - pulled_topo_version = json_pulled_topology["version"] - except KeyError: - logger.debug("Error getting topo version") - continue - - if latest_topology_exists and latest_topo_version == pulled_topo_version: - continue + if not response.ok: + continue + logger.debug("Pulled request from domain controller") + json_pulled_topology = response.json() + pulled_topo_version = json_pulled_topology["version"] + except (requests.ConnectionError, requests.HTTPError): + logger.debug("Error connecting to domain controller...") + continue + except ValueError: + logger.debug("Cannot parse pulled topology, invalid JSON") + continue + except KeyError: + logger.debug("Error getting topo version") + continue + if latest_topology_exists and latest_topo_version == pulled_topo_version: + continue + else: + client_module = importlib.import_module(CLIENT_PACKAGE) + topology_function = getattr(client_module, CLIENT_FUNCTION) + pulled_topology = topology_function() logger.debug("Pulled topo with different version. Adding pulled topo to db") db_instance.add_key_value_pair_to_db( From 88f931ee7915bcfe677d2d1e2b8fa3ea14ef563e Mon Sep 17 00:00:00 2001 From: Yufeng Xin Date: Thu, 23 Oct 2025 10:05:50 -0400 Subject: [PATCH 3/9] integration --- env.template | 5 +++-- pyproject.toml | 1 + sdx_lc/jobs/pull_topo_changes.py | 9 +++++++++ 3 files changed, 13 insertions(+), 2 deletions(-) diff --git a/env.template b/env.template index 27b25e6..4992970 100644 --- a/env.template +++ b/env.template @@ -40,5 +40,6 @@ OXP_PULL_INTERVAL=180 OXP_CONNECTION_URL=http://192.168.201.205:8088/SDX-LC/1.0.0/connection #OXP Client -CLIENT_PACKAGE: sense_sdx -CLIENT_FUNCTION: client \ No newline at end of file +CLIENT_PACKAGE=sense_sdx +CLIENT_FUNCTION=client.py +CLIENT_TOPOLOGY_TRANSLATOR=topology_translate \ No newline at end of file diff --git a/pyproject.toml b/pyproject.toml index 7dbb04a..9d78b3e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -28,6 +28,7 @@ dependencies = [ "asgiref >= 3.7.2", "pymongo > 3.0", "dotenv", + "sense-sdx @ git+https://github.com/atlanticwave-sdx/sense-sdx.git", "sdx-datamodel @ git+https://github.com/atlanticwave-sdx/datamodel@3.0.0.dev5", ] diff --git a/sdx_lc/jobs/pull_topo_changes.py b/sdx_lc/jobs/pull_topo_changes.py index e8cc957..3527216 100644 --- a/sdx_lc/jobs/pull_topo_changes.py +++ b/sdx_lc/jobs/pull_topo_changes.py @@ -1,4 +1,5 @@ import importlib +import importlib.util import json import logging import os.path @@ -18,6 +19,7 @@ CLIENT_PACKAGE = os.getenv("CLIENT_PACKAGE", None) CLIENT_FUNCTION = os.getenv("CLIENT_FUNCTION", None) +CLIENT_TOPOLOGY_TRANSLATOR = os.getenv("CLIENT_TOPOLOGY_TRANSLATOR", None) OXPO_USER = os.environ.get("OXPO_USER", None) OXPO_PASS = os.environ.get("OXPO_PASS", None) @@ -80,8 +82,15 @@ def process_domain_controller_topo(db_instance): continue else: client_module = importlib.import_module(CLIENT_PACKAGE) + path = client_module.__dict__.get('__path__')[0] + spec = importlib.util.spec_from_file_location(CLIENT_TOPOLOGY_TRANSLATOR, path+"/"+CLIENT_FUNCTION) + client = importlib.util.module_from_spec(spec) + spec.loader.exec_module(client) + pulled_topology = client.topology_translate() + topology_function = getattr(client_module, CLIENT_FUNCTION) pulled_topology = topology_function() + json_pulled_topology = json.loads(pulled_topology) logger.debug("Pulled topo with different version. Adding pulled topo to db") db_instance.add_key_value_pair_to_db( From da302f70bd9ee96d3cf9060794219308e2ec8195 Mon Sep 17 00:00:00 2001 From: Yufeng Xin Date: Thu, 23 Oct 2025 11:06:23 -0400 Subject: [PATCH 4/9] formatting --- sdx_lc/app.py | 2 +- sdx_lc/jobs/pull_topo_changes.py | 6 ++++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/sdx_lc/app.py b/sdx_lc/app.py index 5b80de6..da0b670 100644 --- a/sdx_lc/app.py +++ b/sdx_lc/app.py @@ -1,7 +1,6 @@ #!/usr/bin/env python3 import logging -from dotenv import load_dotenv import os import threading from queue import Queue @@ -9,6 +8,7 @@ import connexion from asgiref.wsgi import WsgiToAsgi +from dotenv import load_dotenv from flask import redirect from sdx_datamodel.constants import MessageQueueNames diff --git a/sdx_lc/jobs/pull_topo_changes.py b/sdx_lc/jobs/pull_topo_changes.py index 3527216..b62080c 100644 --- a/sdx_lc/jobs/pull_topo_changes.py +++ b/sdx_lc/jobs/pull_topo_changes.py @@ -82,8 +82,10 @@ def process_domain_controller_topo(db_instance): continue else: client_module = importlib.import_module(CLIENT_PACKAGE) - path = client_module.__dict__.get('__path__')[0] - spec = importlib.util.spec_from_file_location(CLIENT_TOPOLOGY_TRANSLATOR, path+"/"+CLIENT_FUNCTION) + path = client_module.__dict__.get("__path__")[0] + spec = importlib.util.spec_from_file_location( + CLIENT_TOPOLOGY_TRANSLATOR, path + "/" + CLIENT_FUNCTION + ) client = importlib.util.module_from_spec(spec) spec.loader.exec_module(client) pulled_topology = client.topology_translate() From 7c89219615002baad40da8f1b62b491200e2a62c Mon Sep 17 00:00:00 2001 From: Yufeng Xin Date: Thu, 23 Oct 2025 11:26:26 -0400 Subject: [PATCH 5/9] formatting --- sdx_lc/jobs/pull_topo_changes.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/sdx_lc/jobs/pull_topo_changes.py b/sdx_lc/jobs/pull_topo_changes.py index b62080c..d65bec0 100644 --- a/sdx_lc/jobs/pull_topo_changes.py +++ b/sdx_lc/jobs/pull_topo_changes.py @@ -89,9 +89,6 @@ def process_domain_controller_topo(db_instance): client = importlib.util.module_from_spec(spec) spec.loader.exec_module(client) pulled_topology = client.topology_translate() - - topology_function = getattr(client_module, CLIENT_FUNCTION) - pulled_topology = topology_function() json_pulled_topology = json.loads(pulled_topology) logger.debug("Pulled topo with different version. Adding pulled topo to db") From d1d4f7f85762704e20dada3cf4f0cc61636c043d Mon Sep 17 00:00:00 2001 From: Yufeng Xin Date: Thu, 23 Oct 2025 11:43:46 -0400 Subject: [PATCH 6/9] datamodel tag --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 9d78b3e..056f079 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -29,7 +29,7 @@ dependencies = [ "pymongo > 3.0", "dotenv", "sense-sdx @ git+https://github.com/atlanticwave-sdx/sense-sdx.git", - "sdx-datamodel @ git+https://github.com/atlanticwave-sdx/datamodel@3.0.0.dev5", + "sdx-datamodel @ git+https://github.com/atlanticwave-sdx/datamodel@3.0.0.dev13", ] [project.optional-dependencies] From 16245c669403a20bbc5d23c453f2f1e1bc6ff6eb Mon Sep 17 00:00:00 2001 From: Yufeng Xin Date: Mon, 27 Oct 2025 23:24:40 -0400 Subject: [PATCH 7/9] for test --- sdx_lc/app.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/sdx_lc/app.py b/sdx_lc/app.py index da0b670..93d0a01 100644 --- a/sdx_lc/app.py +++ b/sdx_lc/app.py @@ -8,7 +8,6 @@ import connexion from asgiref.wsgi import WsgiToAsgi -from dotenv import load_dotenv from flask import redirect from sdx_datamodel.constants import MessageQueueNames @@ -44,8 +43,6 @@ def create_app(): Create a Flas/Connexion App. """ - load_dotenv() - logger = logging.getLogger(__name__) logging.getLogger("pika").setLevel(logging.WARNING) From 910c813f87b5934651ec8b47257712ef97ccf6af Mon Sep 17 00:00:00 2001 From: Yufeng Xin Date: Tue, 28 Oct 2025 10:09:36 -0400 Subject: [PATCH 8/9] version --- Dockerfile | 2 ++ sdx_lc/jobs/pull_topo_changes.py | 1 + 2 files changed, 3 insertions(+) diff --git a/Dockerfile b/Dockerfile index f049425..4532dfc 100644 --- a/Dockerfile +++ b/Dockerfile @@ -12,6 +12,8 @@ WORKDIR /usr/src/app COPY . /usr/src/app +COPY ./sense-o-auth.yaml.templat[e] /root/.sense-o-auth.yaml + # create a venv. RUN python3 -m venv /opt/venv --upgrade-deps diff --git a/sdx_lc/jobs/pull_topo_changes.py b/sdx_lc/jobs/pull_topo_changes.py index d65bec0..7464ad1 100644 --- a/sdx_lc/jobs/pull_topo_changes.py +++ b/sdx_lc/jobs/pull_topo_changes.py @@ -90,6 +90,7 @@ def process_domain_controller_topo(db_instance): spec.loader.exec_module(client) pulled_topology = client.topology_translate() json_pulled_topology = json.loads(pulled_topology) + json_pulled_topology["version"] = 1 logger.debug("Pulled topo with different version. Adding pulled topo to db") db_instance.add_key_value_pair_to_db( From 6edcefac7d653b0114f87d623d795f6e7ac3f01d Mon Sep 17 00:00:00 2001 From: Yufeng Xin Date: Wed, 29 Oct 2025 17:15:32 -0400 Subject: [PATCH 9/9] exception --- sdx_lc/jobs/pull_topo_changes.py | 24 ++++++++++++++---------- 1 file changed, 14 insertions(+), 10 deletions(-) diff --git a/sdx_lc/jobs/pull_topo_changes.py b/sdx_lc/jobs/pull_topo_changes.py index 7464ad1..f825d7e 100644 --- a/sdx_lc/jobs/pull_topo_changes.py +++ b/sdx_lc/jobs/pull_topo_changes.py @@ -81,16 +81,20 @@ def process_domain_controller_topo(db_instance): if latest_topology_exists and latest_topo_version == pulled_topo_version: continue else: - client_module = importlib.import_module(CLIENT_PACKAGE) - path = client_module.__dict__.get("__path__")[0] - spec = importlib.util.spec_from_file_location( - CLIENT_TOPOLOGY_TRANSLATOR, path + "/" + CLIENT_FUNCTION - ) - client = importlib.util.module_from_spec(spec) - spec.loader.exec_module(client) - pulled_topology = client.topology_translate() - json_pulled_topology = json.loads(pulled_topology) - json_pulled_topology["version"] = 1 + try: + client_module = importlib.import_module(CLIENT_PACKAGE) + path = client_module.__dict__.get("__path__")[0] + spec = importlib.util.spec_from_file_location( + CLIENT_TOPOLOGY_TRANSLATOR, path + "/" + CLIENT_FUNCTION + ) + client = importlib.util.module_from_spec(spec) + spec.loader.exec_module(client) + pulled_topology = client.topology_translate() + json_pulled_topology = json.loads(pulled_topology) + json_pulled_topology["version"] = 1 + except Exception as e: + logger.error("Client Error:{e} \}") + continue logger.debug("Pulled topo with different version. Adding pulled topo to db") db_instance.add_key_value_pair_to_db(