From f2c79738cfa652ae514109c7911fb0ed8f65c479 Mon Sep 17 00:00:00 2001 From: Evgenii Frolov Date: Tue, 5 Feb 2019 18:02:28 +0300 Subject: [PATCH 1/2] Downgrade requirements for Sprut project --- requirements.txt | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/requirements.txt b/requirements.txt index 6e4e6e0..7e19699 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,9 +1,9 @@ -pbr==5.1.1 -webob==1.8.4 -six==1.12.0 -SQLAlchemy==1.0.13 -requests==2.20.0 +pbr==1.8.1 +webob==1.6.1 +six==1.10.0 +SQLAlchemy==1.0.11 +requests==2.6.0 mysql-connector==2.1.3 -oslo.config==6.7.0 --e git+https://github.com/phantomii/bazooka.git@0.0.3#egg=bazooka --e git+https://github.com/phantomii/pyretries.git@0.0.3#egg=pyretries +oslo.config==3.22.0 +-e git+https://github.com/phantomii/bazooka.git@centos-req#egg=bazooka +-e git+https://github.com/phantomii/pyretries.git@centos-req#egg=pyretries From b5f788631785069ba1d420ef4884ff2d89de5517 Mon Sep 17 00:00:00 2001 From: Evgenii Frolov Date: Thu, 17 Jan 2019 14:15:00 +0300 Subject: [PATCH 2/2] REST ORM --- restalchemy/api/resources.py | 19 +- restalchemy/api/routes.py | 7 +- restalchemy/storage/rest/__init__.py | 0 restalchemy/storage/rest/contexts.py | 85 +++++++++ restalchemy/storage/rest/engines.py | 74 ++++++++ restalchemy/storage/rest/orm.py | 268 +++++++++++++++++++++++++++ restalchemy/storage/rest/utils.py | 31 ++++ restalchemy/storage/sql/orm.py | 4 + 8 files changed, 484 insertions(+), 4 deletions(-) create mode 100644 restalchemy/storage/rest/__init__.py create mode 100644 restalchemy/storage/rest/contexts.py create mode 100644 restalchemy/storage/rest/engines.py create mode 100644 restalchemy/storage/rest/orm.py create mode 100644 restalchemy/storage/rest/utils.py diff --git a/restalchemy/api/resources.py b/restalchemy/api/resources.py index 8eb9ed6..5b9616b 100644 --- a/restalchemy/api/resources.py +++ b/restalchemy/api/resources.py @@ -51,7 +51,20 @@ def get_locator(cls, uri): @classmethod def get_resource(cls, request, uri): resource_locator = cls.get_locator(uri) - return resource_locator.get_resource(request, uri) + + # has parent resource? + pstack = resource_locator.path_stack + parent_resource = None + + for pice in reversed(pstack[:-1]): + if not isinstance(pice, six.string_types): + parent_uri = '/'.join(uri.split('/')[:pstack.index(pice) + 2]) + parent_locator = cls.get_locator(parent_uri) + parent_resource = parent_locator.get_resource( + request, parent_uri) + break + + return resource_locator.get_resource(request, uri, parent_resource) @classmethod def set_resource_map(cls, resource_map): @@ -174,8 +187,8 @@ def get_resource_field_name(self, model_field_name): return name.replace('_', '-') if self._convert_underscore else name def is_public_field(self, model_field_name): - return not (model_field_name.startswith('_') or - model_field_name in self._hidden_model_fields) + return not (model_field_name.startswith('_') + or model_field_name in self._hidden_model_fields) def get_model(self): return self._model_class diff --git a/restalchemy/api/routes.py b/restalchemy/api/routes.py index 666c227..dae4baf 100644 --- a/restalchemy/api/routes.py +++ b/restalchemy/api/routes.py @@ -200,8 +200,13 @@ def get_uri(self, model): # FIXME(Eugene Frolov): Header must be string. Not unicode. return str(posixpath.join('/', path)) - def get_resource(self, request, uri): + def get_resource(self, request, uri, parent_resource=None): uuid = posixpath.basename(uri) + if parent_resource: + return (self._controller(request=request) + .get_resource_by_uuid( + uuid=uuid, + parent_resource=parent_resource)) return self._controller(request=request).get_resource_by_uuid( uuid) diff --git a/restalchemy/storage/rest/__init__.py b/restalchemy/storage/rest/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/restalchemy/storage/rest/contexts.py b/restalchemy/storage/rest/contexts.py new file mode 100644 index 0000000..06d0530 --- /dev/null +++ b/restalchemy/storage/rest/contexts.py @@ -0,0 +1,85 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 +# +# Copyright 2018 Mail.ru Group +# +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import bazooka + +from restalchemy.storage.rest import utils + + +class Context(object): + + def __init__(self, request_id=None, request_id_header_name=None, + enable_cache=False): + super(Context, self).__init__() + self._http_client = {} + self._request_id = request_id + self._request_id_header_name = ( + request_id_header_name or 'correlation-id') + self._cache = {} + self._enable_cache = enable_cache + + def _get_client(self, endpoint): + if endpoint in self._http_client: + return self._http_client[endpoint] + cli = bazooka.Client( + correlation_id=self._request_id, + correlation_id_header_name=self._request_id_header_name) + self._http_client[endpoint] = cli + return cli + + def make_post_request(self, endpoint, uri, params): + cli = self._get_client(endpoint) + url = utils.build_collection_uri(endpoint, uri) + response = cli.post(url, json=params) + response_dict = response.json() + if self._enable_cache: + location = response.headers['Location'] + self._cache[location] = response_dict + return response_dict + + def make_put_request(self, endpoint, uri, params): + cli = self._get_client(endpoint) + url = utils.build_resource_uri(endpoint, uri) + response = cli.put(url, json=params) + response_dict = response.json() + if self._enable_cache: + self._cache[url] = response_dict + return response_dict + + def make_list_request(self, endpoint, uri, params): + cli = self._get_client(endpoint) + url = utils.build_collection_uri(endpoint, uri) + response = cli.get(url, params=params) + response_dict = response.json() + return response_dict + + def make_get_request(self, endpoint, uri, params): + url = utils.build_resource_uri(endpoint, uri) + if self._enable_cache and url in self._cache: + return self._cache[url] + cli = self._get_client(endpoint) + response = cli.get(url, params=params) + response_dict = response.json() + if self._enable_cache: + self._cache[url] = response_dict + return response_dict + + def make_delete_request(self, endpoint, uri): + url = utils.build_resource_uri(endpoint, uri) + self._get_client(endpoint).delete(url) + self._cache.pop(url, None) diff --git a/restalchemy/storage/rest/engines.py b/restalchemy/storage/rest/engines.py new file mode 100644 index 0000000..f437286 --- /dev/null +++ b/restalchemy/storage/rest/engines.py @@ -0,0 +1,74 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 +# +# Copyright 2018 Mail.ru Group +# +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from restalchemy.common import singletons + + +class RESTEngine(object): + + def __init__(self, api_endpoint, config=None): + super(RESTEngine, self).__init__() + self._api_endpoint = api_endpoint + self._config = config or {} + + def post(self, uri, params, context): + return context.make_post_request(self._api_endpoint, uri, params) + + def put(self, uri, params, context): + return context.make_put_request(self._api_endpoint, uri, params) + + def list(self, uri, params, context): + return context.make_list_request(self._api_endpoint, uri, params) + + def get(self, uri, params, context): + return context.make_get_request(self._api_endpoint, uri, params) + + def delete(self, uri, context): + return context.make_delete_request(self._api_endpoint, uri) + + +class EngineFactory(singletons.InheritSingleton): + + def __init__(self): + super(EngineFactory, self).__init__() + self._engine = None + self._engines_map = { + 'http': RESTEngine, + 'https': RESTEngine + } + + def configure_factory(self, api_endpoint, config=None): + """Configure_factory + + @property db_url: str. For example driver://user:passwd@host:port/db + """ + schema = api_endpoint.split(':')[0] + try: + self._engine = self._engines_map[schema.lower()](api_endpoint, + config) + except KeyError: + raise ValueError("Can not find driver for schema %s" % schema) + + def get_engine(self): + if self._engine: + return self._engine + raise ValueError("Can not return engine. Please configure " + "EngineFactory") + + +engine_factory = EngineFactory() diff --git a/restalchemy/storage/rest/orm.py b/restalchemy/storage/rest/orm.py new file mode 100644 index 0000000..adb6b39 --- /dev/null +++ b/restalchemy/storage/rest/orm.py @@ -0,0 +1,268 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 +# +# Copyright 2018 Mail.ru Group +# +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import abc + +import six + +from restalchemy.dm import relationships +from restalchemy.storage import base +from restalchemy.storage.rest import engines +from restalchemy.storage.rest import utils + + +class RESTPath(object): + + def __init__(self, *args): + super(RESTPath, self).__init__() + self._rest_path = args + + def _get_parent_name(self, model_cls): + if hasattr(model_cls, 'get_parent_name'): + return model_cls.get_parent_name() + raise NotImplementedError( + "Please implement `get_parent_name` classmethod for class %s" % + model_cls) + + def has_parent(self): + for pice in reversed(self._rest_path): + if not isinstance(pice, six.string_types): + return True + return False + + def get_parent_uri(self, resource_uri): + for pice in reversed(self._rest_path): + if not isinstance(pice, six.string_types): + return '/'.join( + resource_uri.split('/')[:self._rest_path.index(pice) + 2]) + + def get_collection_uri(self, model_cls, obj): + rest_path = [] + curr_model_cls = model_cls + curr_obj = obj + for pice in reversed(self._rest_path): + if isinstance(pice, six.string_types): + rest_path.insert(0, pice) + else: + parent_prop_name = self._get_parent_name(curr_model_cls) + parent = curr_obj[parent_prop_name] + curr_model_cls = type(parent) + curr_obj = parent + rest_path.insert(0, parent.get_resource_id()) + return '/' + utils.force_last_slash('/'.join(rest_path)) + + def get_resource_uri(self, model_cls, obj, obj_id): + return self.get_collection_uri(model_cls, obj) + obj_id + + +class ObjectCollection(base.AbstractObjectCollection): + + @property + def _engine(self): + return engines.engine_factory.get_engine() + + def _filters_to_storage_view(self, context, filters): + # TODO(efrolov): Move this code from class to utils or another + # location. + result = {} + for name, value in (filters or {}).items(): + prop = self.model_cls.properties.properties[name] + if prop.get_property_class() == relationships.Relationship: + result[name] = (self.model_cls.properties.properties[name] + .get_property_type().to_simple_type(context, + value)) + else: + result[name] = (self.model_cls.properties.properties[name] + .get_property_type().to_simple_type(value)) + return result + + def _get_id_property_name(self): + id_property_names = [] + for name, prop in self.model_cls.properties.properties.items(): + if prop.get_property_class().is_id_property(): + id_property_names.append(name) + if len(id_property_names) != 1: + raise ValueError(("Model %s should have only one property mark as " + "`id_property`. Curently: %d") % ( + self.model_cls, len(id_property_names))) + return id_property_names[0] + + def _value_to_simple_type(self, prop_name, prop_value): + return (self.model_cls + .properties + .properties[prop_name] + .get_property_type() + .to_simple_type(prop_value)) + + def get_all(self, context, filters=None): + models_uri = self.model_cls.get_path_ctrl().get_collection_uri( + model_cls=self.model_cls, obj=filters) + resp = self._engine.list(uri=models_uri, + params=self._filters_to_storage_view(context, + filters), + context=context) + for result in resp: + yield self.model_cls.restore_from_storage(context, **result) + + def get_one(self, context, filters=None): + filters = filters.copy() + id_prop_name = self._get_id_property_name() + obj_id = self._value_to_simple_type(id_prop_name, + filters.pop(id_prop_name)) + model_uri = self.model_cls.get_path_ctrl().get_resource_uri( + model_cls=self.model_cls, obj=filters, obj_id=obj_id) + result = self._engine.get( + uri=model_uri, + params=self._filters_to_storage_view(context, filters), + context=context) + return self.model_cls.restore_from_storage(context, **result) + + +@six.add_metaclass(abc.ABCMeta) +class RESTStorableMixin(base.AbstractStorableMixin): + + _saved = False + + _ObjectCollection = ObjectCollection + + @abc.abstractproperty + def __restpath__(self): + raise NotImplementedError() + + @classmethod + def get_path_ctrl(self): + return self.__restpath__ + + @property + def _engine(self): + return engines.engine_factory.get_engine() + + @classmethod + def restore_from_storage(cls, context, **kwargs): + model_format = {} + for name, value in kwargs.items(): + name = name.replace('-', '_') + prop = cls.properties.properties[name] + if prop.get_property_class() == relationships.Relationship: + model_format[name] = ( + prop.get_property_type().from_simple_type(context, + value)) + else: + model_format[name] = ( + prop.get_property_type().from_simple_type(value)) + obj = cls(**model_format) + obj._saved = True + return obj + + def _update_model_from_storage(self, context, **kwargs): + model_format = {} + for name, value in kwargs.items(): + model_name = name.replace('-', '_') + model_format[model_name] = ( + self.properties[model_name].property_type + .from_simple_type(context, value) if isinstance( + self.properties[model_name], + relationships.Relationship) else + self.properties[model_name].property_type + .from_simple_type(value)) + for name, prop in self.properties.items(): + prop.set_value_force(model_format.get(name, None)) + + def _get_prepared_data(self, context, properties=None): + res = {} + props = properties or self.properties + for name, prop in props.items(): + if isinstance(prop, relationships.Relationship): + res[name] = prop.property_type.to_simple_type(context, + prop.value) + else: + res[name] = prop.property_type.to_simple_type(prop.value) + return {name.replace('_', '-'): value for name, value in res.items()} + + def insert(self, context): + models_uri = self.get_path_ctrl().get_collection_uri( + model_cls=type(self), obj=self) + resp = self._engine.post(uri=models_uri, + params=self._get_prepared_data(context), + context=context) + self._saved = True + self._update_model_from_storage(context, **resp) + + def save(self, context): + self.update(context) if self._saved else self.insert(context) + + def update(self, context): + model_uri = self.get_path_ctrl().get_resource_uri( + model_cls=type(self), obj=self, obj_id=self.get_resource_id()) + dirty_props = {name: prop for name, prop in self.properties.items() + if prop.is_dirty()} + result = self._engine.put( + uri=model_uri, + params=self._get_prepared_data(context, properties=dirty_props), + context=context) + return self.restore_from_storage(context, **result) + + def delete(self, context): + model_uri = self.get_path_ctrl().get_resource_uri( + model_cls=type(self), obj=self, obj_id=self.get_resource_id()) + self._engine.delete(uri=model_uri, context=context) + self._saved = False + + def _get_resource_id_prop(self): + for prop in self.properties.values(): + if prop.is_id_property(): + return prop + raise ValueError("Model (%s) should contain a property of IdProperty " + "type" % self) + + def get_resource_id(self): + prop = self._get_resource_id_prop() + return prop.property_type.to_simple_type(self.get_id()) + + def get_resource_uri(self): + prop = self._get_resource_id_prop() + return self.get_path_ctrl().get_resource_uri( + model_cls=type(self), + obj=self, + obj_id=prop.property_type.to_simple_type(self.get_id())) + + @classmethod + def to_simple_type(cls, context, value): + return value.get_resource_uri() if value is not None else None + + @classmethod + def from_simple_type(cls, context, value): + if value is None: + return None + raw_value = value.split('/')[-1] + path_ctrl = cls.get_path_ctrl() + filters = {} + for name, prop in cls.properties.items(): + if prop.is_id_property(): + if path_ctrl.has_parent(): + parent_name = cls.get_parent_name() + parent_prop = (cls.properties.properties[parent_name] + .get_property_type()) + parent = parent_prop.from_simple_type( + context, path_ctrl.get_parent_uri(value)) + filters[parent_name] = parent + value = (cls.properties.properties[name].get_property_type() + .from_simple_type(raw_value)) + filters[name] = value + return cls.objects.get_one(context, filters=filters) + raise NotImplementedError("Id property for %s not found!!!" % cls) diff --git a/restalchemy/storage/rest/utils.py b/restalchemy/storage/rest/utils.py new file mode 100644 index 0000000..2327d25 --- /dev/null +++ b/restalchemy/storage/rest/utils.py @@ -0,0 +1,31 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 +# +# Copyright 2018 Mail.ru Group +# +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from six.moves.urllib import parse as urllib + + +def force_last_slash(path): + return path if path[-1] == '/' else path + '/' + + +def build_collection_uri(endpoint, uri): + return urllib.urljoin(endpoint, force_last_slash(uri)) + + +def build_resource_uri(endpoint, uri): + return urllib.urljoin(endpoint, uri) diff --git a/restalchemy/storage/sql/orm.py b/restalchemy/storage/sql/orm.py index 4726dda..a79af48 100644 --- a/restalchemy/storage/sql/orm.py +++ b/restalchemy/storage/sql/orm.py @@ -202,6 +202,8 @@ def delete(self, session=None): @classmethod def to_simple_type(cls, value): + if value is None: + return None for prop in value.properties.values(): if prop.is_id_property(): return prop.property_type.to_simple_type(value.get_id()) @@ -210,6 +212,8 @@ def to_simple_type(cls, value): @classmethod def from_simple_type(cls, value): + if value is None: + return None for name, prop in cls.properties.items(): if prop.is_id_property(): value = (cls.properties.properties[name].get_property_type()