From a1d4aa66bc6b3de041ec6e4eb4fe40383945510b Mon Sep 17 00:00:00 2001 From: Daniel Pyrathon Date: Fri, 30 Nov 2018 09:59:37 -0800 Subject: feat(order_utils.py): schema resolver cache (#1317) * Implemented basic functionality for using cache layer of LocalRefResolver * Use `importlib` instead of `imp`, since it's been deprecated. Legacy `load_module()` reloads modules even if they are already imported, causing tests to fail when run in non-deterministic ordering, so we replace it with `import_module()` --- .../src/zero_ex/json_schemas/__init__.py | 81 +++++++++++++--------- .../order_utils/stubs/jsonschema/__init__.pyi | 10 ++- python-packages/order_utils/test/test_doctest.py | 5 +- .../order_utils/test/test_json_schemas.py | 23 ++++++ 4 files changed, 81 insertions(+), 38 deletions(-) create mode 100644 python-packages/order_utils/test/test_json_schemas.py (limited to 'python-packages/order_utils') diff --git a/python-packages/order_utils/src/zero_ex/json_schemas/__init__.py b/python-packages/order_utils/src/zero_ex/json_schemas/__init__.py index 2a1728b8a..a76a2fa3b 100644 --- a/python-packages/order_utils/src/zero_ex/json_schemas/__init__.py +++ b/python-packages/order_utils/src/zero_ex/json_schemas/__init__.py @@ -8,6 +8,51 @@ from pkg_resources import resource_string import jsonschema +class _LocalRefResolver(jsonschema.RefResolver): + """Resolve package-local JSON schema id's.""" + + def __init__(self): + """Initialize a new instance.""" + self.ref_to_file = { + "/addressSchema": "address_schema.json", + "/hexSchema": "hex_schema.json", + "/orderSchema": "order_schema.json", + "/wholeNumberSchema": "whole_number_schema.json", + "/ECSignature": "ec_signature_schema.json", + "/signedOrderSchema": "signed_order_schema.json", + "/ecSignatureParameterSchema": ( + "ec_signature_parameter_schema.json" + "" + ), + } + jsonschema.RefResolver.__init__(self, "", "") + + def resolve_from_url(self, url: str) -> str: + """Resolve the given URL. + + :param url: a string representing the URL of the JSON schema to fetch. + :returns: a string representing the deserialized JSON schema + :raises jsonschema.ValidationError: when the resource associated with + `url` does not exist. + """ + ref = url.replace("file://", "") + if ref in self.ref_to_file: + return json.loads( + resource_string( + "zero_ex.json_schemas", f"schemas/{self.ref_to_file[ref]}" + ) + ) + raise jsonschema.ValidationError( + f"Unknown ref '{ref}'. " + + f"Known refs: {list(self.ref_to_file.keys())}." + ) + + +# Instantiate the `_LocalRefResolver()` only once so that `assert_valid()` can +# perform multiple schema validations without reading from disk the schema +# every time. +_LOCAL_RESOLVER = _LocalRefResolver() + + def assert_valid(data: Mapping, schema_id: str) -> None: """Validate the given `data` against the specified `schema`. @@ -24,38 +69,6 @@ def assert_valid(data: Mapping, schema_id: str) -> None: ... ) """ # noqa - class LocalRefResolver(jsonschema.RefResolver): - """Resolve package-local JSON schema id's.""" - - def __init__(self): - self.ref_to_file = { - "/addressSchema": "address_schema.json", - "/hexSchema": "hex_schema.json", - "/orderSchema": "order_schema.json", - "/wholeNumberSchema": "whole_number_schema.json", - "/ECSignature": "ec_signature_schema.json", - "/ecSignatureParameterSchema": ( - "ec_signature_parameter_schema.json" + "" - ), - } - jsonschema.RefResolver.__init__(self, "", "") - - def resolve_from_url(self, url): - """Resolve the given URL.""" - ref = url.replace("file://", "") - if ref in self.ref_to_file: - return json.loads( - resource_string( - "zero_ex.json_schemas", - f"schemas/{self.ref_to_file[ref]}", - ) - ) - raise jsonschema.ValidationError( - f"Unknown ref '{ref}'. " - + f"Known refs: {list(self.ref_to_file.keys())}." - ) - resolver = LocalRefResolver() - jsonschema.validate( - data, resolver.resolve_from_url(schema_id), resolver=resolver - ) + _, schema = _LOCAL_RESOLVER.resolve(schema_id) + jsonschema.validate(data, schema, resolver=_LOCAL_RESOLVER) diff --git a/python-packages/order_utils/stubs/jsonschema/__init__.pyi b/python-packages/order_utils/stubs/jsonschema/__init__.pyi index 762b58b22..442e2f65e 100644 --- a/python-packages/order_utils/stubs/jsonschema/__init__.pyi +++ b/python-packages/order_utils/stubs/jsonschema/__init__.pyi @@ -1,5 +1,11 @@ -from typing import Any, Dict +from typing import Any, Dict, Tuple -class RefResolver: pass + +class RefResolver: + def resolve(self, url: str) -> Tuple[str, Dict]: + ... + + +class ValidationError(Exception): pass def validate(instance: Any, schema: Dict, cls=None, *args, **kwargs) -> None: pass diff --git a/python-packages/order_utils/test/test_doctest.py b/python-packages/order_utils/test/test_doctest.py index f692b3b6c..297f75e75 100644 --- a/python-packages/order_utils/test/test_doctest.py +++ b/python-packages/order_utils/test/test_doctest.py @@ -2,16 +2,17 @@ from doctest import testmod import pkgutil +import importlib import zero_ex def test_all_doctests(): """Gather zero_ex.* modules and doctest them.""" - for (importer, modname, _) in pkgutil.walk_packages( + for (_, modname, _) in pkgutil.walk_packages( path=zero_ex.__path__, prefix="zero_ex." ): - module = importer.find_module(modname).load_module(modname) + module = importlib.import_module(modname) print(module) (failure_count, _) = testmod(module) assert failure_count == 0 diff --git a/python-packages/order_utils/test/test_json_schemas.py b/python-packages/order_utils/test/test_json_schemas.py new file mode 100644 index 000000000..51cecbd4f --- /dev/null +++ b/python-packages/order_utils/test/test_json_schemas.py @@ -0,0 +1,23 @@ +"""Tests of zero_ex.json_schemas""" + + +from zero_ex.order_utils import make_empty_order +from zero_ex.json_schemas import _LOCAL_RESOLVER, assert_valid + + +def test_assert_valid_caches_resources(): + """Test that the JSON ref resolver in `assert_valid()` caches resources + + In order to test the cache we much access the private class of + `json_schemas` and reset the LRU cache on `_LocalRefResolver`. + For this to happen, we need to disable errror `W0212` + on _LOCAL_RESOLVER + """ + _LOCAL_RESOLVER._remote_cache.cache_clear() # pylint: disable=W0212 + + assert_valid(make_empty_order(), "/orderSchema") + cache_info = ( + _LOCAL_RESOLVER._remote_cache.cache_info() # pylint: disable=W0212 + ) + assert cache_info.currsize == 4 + assert cache_info.hits == 10 -- cgit v1.2.3