feat: upgrade openapi-core and pyramid-openapi3 dependencies

Also upgrade from openapi version 3.0.3 to 3.1.0

- nullables are replaced by the "null" type.
- application/octet-stream content schema don't require specifying a schema

cubicweb_api/__pkginfo__.py

@@ -32,8 +32,8 @@
 __depends__ = {
     "cubicweb": ">= 3.36.0,<5",
     "PyJWT": ">= 2.4.0",
-    "pyramid-openapi3": ">= 0.15",
-    "openapi-core": ">= 0.16.1, < 0.17.0",  # to be removed when pyramid-openapi3 will be compatible
+    "pyramid-openapi3": ">= 0.19",
+    "openapi-core": ">= 0.17.0",  # to be removed when pyramid-openapi3 will be compatible
 }
 __recommends__ = {}
 

cubicweb_api/openapi/openapi.py

@@ -119,15 +119,6 @@
     config.add_exception_view(
         view=custom_openapi_validation_error, context=ResponseValidationError
     )
-    # See https://github.com/python-openapi/openapi-core/issues/630
-    # openapi_core's default multipart deserializer does not work as it expects the Content-Type
-    # header to be part of the request (uses email's multipart parser)
-    # pyramid_openapi3 already deserializes the request using self.request.POST.mixed()
-    # which returns a python object
-    # So our custom deserializer is a simple identity
-    config.registry.settings["pyramid_openapi3_deserializers"] = {
-        "multipart/form-data": lambda x: x
-    }
 
 
 def custom_openapi_validation_error(

cubicweb_api/openapi/openapi_template.yaml

@@ -43,10 +43,7 @@
         '200':
           description: The binary data
           content:
-            application/octet-stream:
-              schema:
-                type: string
-                format: binary
+            application/octet-stream: {}
         '400':
           description: The parameters are not valid
           content:
@@ -73,7 +70,21 @@
               type: array
               items:
                 $ref: '#/components/schemas/RqlTransactionParams'
-          multipart/form-data: {}
+          multipart/form-data:
+            schema:
+              type: object
+              properties:
+                queries:
+                  allOf:
+                    - type: array
+                      items:
+                        $ref: '#/components/schemas/RqlTransactionParams'
+              additionalProperties:
+                type: string
+              required: ["queries"]
+            encoding:
+              queries:
+                contentType: "application/json"
       responses:
         '200':
           description: The RQL result set
@@ -154,7 +165,7 @@
             application/json:
               schema:
                 $ref: '#/components/schemas/SiteInfo'
-openapi: 3.0.3
+openapi: 3.1.0
 components:
   parameters:
     client:
@@ -196,6 +207,7 @@
               cardinality:
                 type: string
               constraints:
-                type: array
-                nullable: true
+                type:
+                  - "array"
+                  - "null"
               default:
@@ -201,6 +213,7 @@
               default:
-                type: string
-                nullable: true
+                type:
+                  - "string"
+                  - "null"
     RqlTransactionParams:
       type: object
       properties:
@@ -216,4 +229,5 @@
         - query
     RqlTransactionParamsValue:
       oneOf:
+        - type: "null"
         - type: string
@@ -219,5 +233,4 @@
         - type: string
-          nullable: true
         - type: number
         - type: boolean
         - type: object
@@ -286,8 +299,8 @@
             type: array
             items:
               oneOf:
-                - type: 'string'
-                  nullable: true
-                - type: 'number'
-                - type: 'boolean'
+                - type: "null"
+                - type: "string"
+                - type: "number"
+                - type: "boolean"
     CurrentUser:
@@ -293,17 +306,18 @@
     CurrentUser:
-      type: object
-      nullable: true
-      properties:
-        eid:
-          type: integer
-        login:
-          type: string
-        dcTitle:
-          type: string
-      required:
-        - eid
-        - login
-        - dcTitle
+      oneOf:
+        - type: "null"
+        - type: object
+          properties:
+            eid:
+              type: integer
+            login:
+              type: string
+            dcTitle:
+              type: string
+          required:
+            - eid
+            - login
+            - dcTitle
     SiteInfo:
       type: object
       properties:
@@ -325,6 +339,7 @@
                   items:
                     type: integer
                 datadir_url:
-                  type: string
-                  nullable: true
+                  type:
+                    - "null"
+                    - "string"
                 instance_home:
@@ -330,5 +345,7 @@
                 instance_home:
-                  type: string
+                  type:
+                    - "null"
+                    - "string"
               type: object
             stats:
               type: object

cubicweb_api/routes.py

@@ -14,7 +14,6 @@
 # You should have received a copy of the GNU Lesser General Public License
 # along with this program. If not, see <https://www.gnu.org/licenses/>.
 
-import json
 import logging
 from enum import Enum
 from functools import partial
@@ -18,7 +17,6 @@
 import logging
 from enum import Enum
 from functools import partial
-from os import path
 
 from cubicweb import Binary
 from cubicweb._exceptions import UnknownEid
@@ -27,9 +25,5 @@
 from cubicweb.rset import ResultSet
 from cubicweb.schema_exporters import JSONSchemaExporter
 from cubicweb.sobjects.services import StatsService, GcStatsService
-from openapi_core import Spec
-from openapi_core import validate_request
-from openapi_spec_validator import validate_spec
-from openapi_spec_validator.readers import read_from_filename
 from pyramid.config import Configurator
 from pyramid.request import Request
@@ -34,7 +28,5 @@
 from pyramid.config import Configurator
 from pyramid.request import Request
-from pyramid_openapi3.exceptions import RequestValidationError
-from pyramid_openapi3.wrappers import PyramidOpenAPIRequest
 from yams.schema import RelationDefinitionSchema
 from yams.types import DefinitionName
 
@@ -42,7 +34,7 @@
     API_ROUTE_NAME_PREFIX,
 )
 from cubicweb_api.httperrors import get_http_error
-from cubicweb_api.openapi.openapi import setup_openapi, custom_openapi_validation_error
+from cubicweb_api.openapi.openapi import setup_openapi
 from cubicweb_api.transaction import Transaction, BinaryResolver
 from cubicweb_api.util import get_cw_repo
 
@@ -124,14 +116,5 @@
     """
     body = request.openapi_validated.body
 
-    # As we can't validate params in the multipart, we convert the request to JSON
-    # and manually validate it
-    request_as_json = request.copy()
-    request_as_json.content_type = "application/json"
-    request_as_json.body = body["queries"].encode()
-
-    spec_path = path.join(request.cw_cnx.repo.config.apphome, "openapi.yaml")
-    spec_dict, _ = read_from_filename(spec_path)
-    validate_spec(spec_dict)
-    spec = Spec.create(spec_dict)
+    queries = body["queries"]
 
@@ -137,15 +120,10 @@
 
-    # Manually run the request validation
-    openapi_request = PyramidOpenAPIRequest(request_as_json)
-    try:
-        validate_request(request=openapi_request, spec=spec)
-    except Exception as e:
-        return custom_openapi_validation_error(
-            RequestValidationError(errors=[e]), request
-        )
-
-    queries = json.loads(body["queries"])
-    transaction = Transaction(queries, BinaryResolver(body))
+    # XXX: `queries" property is correctly parsed as an array of records but
+    # openapi-core doesn't support `schema.additionalProperties` which we would
+    # normally use to check form-data fields corresponding to binary params.
+    # A workaround is to get these binary params from the pyramid request
+    # itself instead of retrieving them from the validated body.
+    transaction = Transaction(queries, BinaryResolver(request.params))
     return _transaction_result_to_json(transaction.execute(request.cw_cnx))
 
 

test/test_api_binary.py

@@ -121,8 +121,8 @@
                 {
                     "exception": "MissingRequiredParameter",
                     "field": "eid",
-                    "message": "Missing required parameter: eid",
+                    "message": "Missing required query parameter: eid",
                 },
                 {
                     "exception": "MissingRequiredParameter",
                     "field": "attribute",
@@ -125,8 +125,8 @@
                 },
                 {
                     "exception": "MissingRequiredParameter",
                     "field": "attribute",
-                    "message": "Missing required parameter: attribute",
+                    "message": "Missing required query parameter: attribute",
                 },
             ],
             "message": "Your request could not be validated against the openapi "
@@ -143,7 +143,8 @@
         assert response == {
             "data": [
                 {
-                    "exception": "CastError",
+                    "exception": "ParameterValidationError",
+                    "field": "eid",
                     "message": "Failed to cast value to integer type: this is wrong",
                 }
             ],
@@ -162,6 +163,7 @@
             "data": [
                 {
                     "exception": "ValidationError",
+                    "field": "attribute",
                     "message": "'5' does not match '^[a-z_][a-z0-9_]+$'",
                 }
             ],

test/test_api_login.py

@@ -118,7 +118,7 @@
         assert response == {
             "data": [
                 {
-                    "exception": "MediaTypeNotFound",
+                    "exception": "RequestBodyValidationError",
                     "message": "Content for the following mimetype not found: "
                     "text/plain. Valid mimetypes: ['application/json']",
                 }

test/test_api_rql.py

@@ -97,17 +97,7 @@
             status=400,
         ).json
 
-        assert response == {
-            "data": [
-                {
-                    "exception": "MissingRequiredRequestBody",
-                    "message": "Missing required request body",
-                }
-            ],
-            "message": "Your request could not be validated against the openapi "
-            "specification.",
-            "title": "OpenApiValidationError",
-        }
+        assert "required" in response["data"][0]["message"]
 
     def test_401_error_on_rql_when_not_authenticated(self):
         queries = [
@@ -233,19 +223,18 @@
         ]
         response = self.make_rql_request(queries, 400)
 
-        assert response == {
-            "data": [
-                {
-                    "exception": "ValidationError",
-                    "message": "{'type': 'query_reference', 'queryIndex': 'not a number', "
-                    "'row': 0, 'column': 0} is not "
-                    "valid under any of the given schemas",
-                }
-            ],
-            "message": "Your request could not be validated against the openapi "
-            "specification.",
-            "title": "OpenApiValidationError",
-        }
+        assert response["message"] == (
+            "Your request could not be validated against the openapi " "specification."
+        )
+        assert response["title"] == "OpenApiValidationError"
+
+        data = response["data"][0]
+        assert data["message"] == (
+            "{'type': 'query_reference', 'queryIndex': 'not a number', "
+            "'row': 0, 'column': 0} is not "
+            "valid under any of the given schemas"
+        )
+        assert data["exception"] == "ValidationError"
 
     def test_missing_custom_headers_returns_400(self):
         response = self.webapp.post(
@@ -356,6 +345,7 @@
             "data": [
                 {
                     "exception": "ValidationError",
+                    "field": "queries/0/params/binary_ref",
                     "message": "{'type': 'binary_reference'} is valid under each of "
                     "{'additionalProperties': False, 'properties': {'ref': "
                     "{'type': 'string'}, 'type': {'type': 'string'}}, "