Update dependencies (#49)

* Update nunavut 2.3.1, celery 5.3.4, flask 2.3.3, leaving minio because of #48 (#49) * Update nunaserver/{generator.py,tasks.py} along with new dependencies (#49) * Remove nunavut._generators (#49)
OpenCyphal · Nov 6, 2023 · ac01b47 · ac01b47
1 parent b75e096
commit ac01b47
Show file tree

Hide file tree

Showing 3 changed files with 47 additions and 48 deletions.
diff --git a/nunaserver/nunaserver/generator.py b/nunaserver/nunaserver/generator.py
@@ -6,17 +6,14 @@
 import os
 import tempfile
 import zipfile
-import logging
+import minio
+import typing
 from typing import List
 from pathlib import Path
 from pydsdl import read_namespace
 from pydsdl._error import InvalidDefinitionError
-import nunavut
-import minio
 from nunavut import build_namespace_tree
-from nunavut.generators import create_generators
-from nunavut.lang import LanguageContext
-from nunavut.jinja import DSDLCodeGenerator
+from nunavut import DSDLCodeGenerator, SupportGenerator, AbstractGenerator, Namespace
 from nunaserver import settings
 from nunaserver.tasks import celery
 from nunaserver.logging import init_logging
@@ -26,9 +23,16 @@
     fetch_remote_namespace,
     unzip_to_directory,
 )
+from nunavut.lang import (
+    Language,
+    LanguageContextBuilder,
+)
 
 init_logging()
 
+def create_default_generators(ns: Namespace, **kwargs: typing.Any) -> tuple[AbstractGenerator, AbstractGenerator]:
+    return DSDLCodeGenerator(ns, **kwargs), SupportGenerator(ns, **kwargs)
+
 # pylint: disable=too-many-locals,too-many-arguments
 @celery.task(bind=True)
 def generate_dsdl(
@@ -80,13 +84,9 @@ def generate_dsdl(
     inner = [d for d in Path(arch_dir).iterdir() if d.is_dir()]
     namespaces = []
     for path in inner:
-        subnss = [
-            d for d in path.iterdir() if d.is_dir() and not d.name.startswith(".")
-        ]
+        subnss = [d for d in path.iterdir() if d.is_dir() and not d.name.startswith(".")]
         if len(subnss) > 0:
-            namespaces.extend(
-                [d for d in path.iterdir() if d.is_dir() and not d.name.startswith(".")]
-            )
+            namespaces.extend([d for d in path.iterdir() if d.is_dir() and not d.name.startswith(".")])
         else:
             namespaces.append(path)
 
@@ -106,9 +106,7 @@ def generate_dsdl(
         command += f" dsdl_src{str(ns_dir).replace(str(arch_dir), '')}"
         for lookup_dir in namespaces:
             if lookup_dir != ns_dir:
-                command += (
-                    f" --lookup dsdl_src{str(lookup_dir).replace(str(arch_dir), '')}"
-                )
+                command += f" --lookup dsdl_src{str(lookup_dir).replace(str(arch_dir), '')}"
 
     self.update_state(
         state="PROGRESS",
@@ -138,36 +136,30 @@ def generate_dsdl(
         extra_includes = list(map(str, extra_includes))
 
         try:
-            compound_types = read_namespace(
-                namespace, extra_includes, allow_unregulated_fixed_port_id=False
-            )
+            compound_types = read_namespace(namespace, extra_includes, allow_unregulated_fixed_port_id=False)
         except InvalidDefinitionError as error:
             text = str(error).replace(str(arch_dir), "")
             raise RuntimeError(f"{text}") from error
 
         # Select target language and configure context
-        language_options = {}
-        language_options["target_endianness"] = target_endian
-        language_options["omit_float_serialization_support"] = (
-            "--omit-float-serialization-support" in flags
-        )
-        language_options["enable_serialization_asserts"] = (
-            "--enable-serialization-asserts" in flags
-        )
-        lang_context = LanguageContext(
-            target_lang,
-            omit_serialization_support_for_target="--omit-serialization-support"
-            in flags,
-            language_options=language_options,
-        )
+        language_options = {
+            "target_endianness": target_endian,
+            "omit_float_serialization_support": ("--omit-float-serialization-support" in flags),
+            "enable_serialization_asserts": ("--enable-serialization-asserts" in flags),
+            "enable_override_variable_array_capacity": ("--enable-override-variable-array-capacity" in flags),
+        }
 
-        # Build namespace tree
-        root_namespace = build_namespace_tree(
-            compound_types, namespace, out_dir, lang_context
+        language_context = (
+            LanguageContextBuilder(include_experimental_languages=True)
+            .set_target_language(target_lang)
+            .set_target_language_configuration_override(Language.WKCV_LANGUAGE_OPTIONS, language_options)
+            .create()
         )
 
+        root_namespace = build_namespace_tree(compound_types, namespace, str(out_dir), language_context)
+
         # Generate code
-        generator, support_generator = create_generators(root_namespace)
+        generator, support_generator = create_default_generators(root_namespace)
         generator.generate_all()
         support_generator.generate_all()
 
@@ -186,17 +178,15 @@ def generate_dsdl(
                     "docs",
                     str(rel_path),
                     str(file.absolute()),
-                    content_type="text/html"
+                    content_type="text/html",
                 )
         return {
             "current": len(namespaces),
             "total": len(namespaces),
             "command": command,
             "type": "htmldoc",
             "status": "Complete!",
-            "result": [
-                f"{settings.MINIO_DOCS}/{doc_url}/{str(ns).split('/')[-1]}/index.html" for ns in namespaces
-            ],
+            "result": [f"{settings.MINIO_DOCS}/{doc_url}/{str(ns).split('/')[-1]}/index.html" for ns in namespaces],
         }
     else:
         # Zip result

diff --git a/nunaserver/nunaserver/tasks.py b/nunaserver/nunaserver/tasks.py
@@ -2,6 +2,17 @@
 Run Nunavut jobs as Celery background tasks
 to allow for long-running jobs.
 """
+# see issue https://github.com/celery/kombu/issues/1804
+import functools
+from threading import RLock
+import kombu.utils
+
+if not getattr(kombu.utils.cached_property, 'lock', None):
+    setattr(kombu.utils.cached_property, 'lock', functools.cached_property(lambda _: RLock()))
+    # Must call __set_name__ here since this cached property is not defined in the context of a class
+    # Refer to https://docs.python.org/3/reference/datamodel.html#object.__set_name__
+    kombu.utils.cached_property.lock.__set_name__(kombu.utils.cached_property, 'lock')
+
 from celery import Celery, Task
 from nunaserver import settings
 
@@ -11,9 +22,7 @@ def make_celery(name):
     Create a Celery wrapper for use in running
     background tasks.
     """
-    celery_inst = Celery(
-        name, backend=settings.CELERY_RESULT_BACKEND, broker=settings.CELERY_BROKER_URL
-    )
+    celery_inst = Celery(name, backend=settings.CELERY_RESULT_BACKEND, broker=settings.CELERY_BROKER_URL)
 
     return celery_inst
 

diff --git a/nunaserver/requirements.txt b/nunaserver/requirements.txt
@@ -1,5 +1,5 @@
 # Flask for serving (obviously) + CORS api
-flask==2.0.1
+flask==2.3.3
 flask-cors==3.0.10
 flask-limiter==1.4
 gunicorn==20.1.0
@@ -8,12 +8,12 @@ gunicorn==20.1.0
 requests==2.25.1
 
 # Celery
-celery==5.1.0
-redis==3.5.3
+celery==5.3.4
+redis==5.0.1
 
 # Nunavut
-nunavut==1.3.0
-pydsdl==1.12.1
+nunavut==2.3.1
+pydsdl==1.20.1
 
 # MinIO
 minio==7.0.3