librelist archives

Re: Lazily load class based views

From:

Sean Lynch

Date:

2011-09-18 @ 04:40

I figured it out.  I needed to adjust the import_name for class based views,
not just the __module__ and __name__.

My implementation could use some cleanup, but it works.



import logging
import re
from werkzeug.utils import import_string, cached_property

class LazyView(object):

    def __init__(self, import_name):

        import_name_list = import_name.split('.')
        if ("as_view" in import_name_list[-1]):
            # Class view
            self.__module__ = '.'.join(import_name_list[:-2]) # Remove class
name and as_view
            self.__name__ = import_name_list[-2] # Use class name
            self.import_name = '.'.join(import_name_list[:-1]) # all but
as_view(...)

            params_string = import_name_list[-1] # as_view(...)
            params_list = re.findall(r'as_view\((.*)\)', params_string)[0]
            self.params = [x.strip().replace('\"', '') for x in
params_list.split(',')]
        else:
            # Function view
            self.__module__, self.__name__ = import_name.rsplit('.', 1)
            self.import_name = import_name

    @cached_property
    def view(self):
        logging.info("Lazy loading '%s' for first time" % self.import_name)

        view = import_string(self.import_name)
        if hasattr(self, 'params'):
            # Class view
            view = view.as_view(*self.params)

        return view

    def __call__(self, *args, **kwargs):
        return self.view(*args, **kwargs)


On Sat, Sep 17, 2011 at 12:36 AM, Sean Lynch <techniq35@gmail.com> wrote:

> I've been trying to follow the lazy loading of views pattern (
> http://flask.pocoo.org/docs/patterns/lazyloading/) to reduce my instance
> startup time on Google App Engine, but not matter what I try, I also get the
> following error when trying to access a class based view (works fine for
> function based views).  (Note: I've modified the examples from the actual
> names to simplify the issue, but I can provide actual code examples if
> needed)
>
>     "ImportError: No module named MyView"
>
> Here is how the url is registered:
>     def url(rule, endpoint=None, import_name=None, **options):
>         view = LazyView('application.admin.' + import_name)
>         bp.add_url_rule(rule, endpoint, view, **options)
>
>     ...
>     url('/myview/', 'myview', 'views.slide.MyView.as_view("myview")')
>     ...
>
>
> and finally, here is the implementation of LazyView.  I intially tried what
> is in the patterns docs, but tried a few things with setting the __module__
> and __name__ differently, to no avail.
>
>     import logging
>     from werkzeug.utils import import_string, cached_property
>
>     class LazyView(object):
>
>         def __init__(self, import_name):
>
>             import_name_list = import_name.split('.')
>             if ("as_view" in import_name_list[-1]):
>                 self.__module__ = '.'.join(import_name_list[:-1])  # Remove
> class name and as_view
>                 self.__name__ = import_name_list[-2]               # Use
> class name
>             else:
>                 self.__module__, self.__name__ = import_name.rsplit('.', 1)
>
>             self.import_name = import_name
>
>         @cached_property
>         def view(self):
>             logging.info("Lazy loading '%s' for first time" %
> self.import_name)
>             return import_string(self.import_name)
>
>         def __call__(self, *args, **kwargs):
>             return self.view(*args, **kwargs)
>
>
>
>
>
> Here is the full stacktrace:
>
> File
> 
"C:\Users\smlynch\Dev\Projects\freelance\wchd_flask\packages/flask.zip\flask\app.py",
> line 1306, in __call__
> File
> 
"C:\Users\smlynch\Dev\Projects\freelance\wchd_flask\packages/flask.zip\flask\app.py",
> line 1294, in wsgi_app
> File
> 
"C:\Users\smlynch\Dev\Projects\freelance\wchd_flask\packages/flask.zip\flask\app.py",
> line 1292, in wsgi_app
> File
> 
"C:\Users\smlynch\Dev\Projects\freelance\wchd_flask\packages/flask.zip\flask\app.py",
> line 1062, in full_dispatch_request
> File
> 
"C:\Users\smlynch\Dev\Projects\freelance\wchd_flask\packages/flask.zip\flask\app.py",
> line 1060, in full_dispatch_request
> File
> 
"C:\Users\smlynch\Dev\Projects\freelance\wchd_flask\packages/flask.zip\flask\app.py",
> line 1047, in dispatch_request
> File
> "C:\Users\smlynch\Dev\Projects\freelance\wchd_flask\packages\flaskext\views.py",
> line 31, in __call__
> return self.view(*args, **kwargs)
> File
> "C:\Users\smlynch\Dev\Projects\freelance\wchd_flask\packages\werkzeug\utils.py",
> line 79, in __get__
> value = self.func(obj)
> File
> "C:\Users\smlynch\Dev\Projects\freelance\wchd_flask\packages\flaskext\views.py",
> line 28, in view
> return import_string(self.import_name)
> File
> "C:\Users\smlynch\Dev\Projects\freelance\wchd_flask\packages\werkzeug\utils.py",
> line 526, in import_string
> return getattr(__import__(module, None, None, [obj]), obj)
> ImportError: No module named MyView
>
>
>
> Any have ideas?
>

Hi,

Why do do it like this?


from threading import Lock
from werkzeug import import_string

class LazyView(object):

    def __init__(self, import_name, *args, **kwargs):
        self.__module__, self.__name__ = import_name.rsplit('.', 1)
        self.import_name = import_name
        self.args = args
        self.kwargs = kwargs
        self.lock = Lock()
        self._view = None

    def view(self):
        if self._view is not None:
            return self._view
        with self.lock:
            if self._view is None:
                view_cls = import_string(self.import_name)
                self._view = view_cls(*self.args, **self.kwargs)
            return self._view

    def __call__(self, *args, **kwargs):
        return self.view(*args, **kwargs)


view = LazyView('application.admin.FooView')


Regards,
Armin

It looks like this LazyView only works with class based views, so I would
need to use the one on the patterns page for function based views (
http://flask.pocoo.org/docs/patterns/lazyloading/)?

Your implementation is much cleaner.  I didn't realize you could init the
class instead of using as_view.  I guess that is way you have to use the
thread lock, since you're not init'ing a new instance for each request, but
using one instance as a callable.

I was using the helper function to easily map urls:

def url(rule, endpoint=None, import_name=None, **options):
    view = LazyView('application.' + import_name)
    app.add_url_rule(rule, endpoint, view, **options)

So I guess I'll need to have 2 helper methods and use the appropriate
LazyView implementation.


As a side note, what do you think about extending @cached_property in
Werkzeug to take in a "with_lock = True" parameter, so the LazyView could
become:

from werkzeug.utils import cached_property, import_string

class LazyView(object):

   def __init__(self, import_name, *args, **kwargs):
       self.__module__, self.__name__ = import_name.rsplit('.', 1)
       self.import_name = import_name
       self.args = args
       self.kwargs = kwargs

    @cached_property(with_lock=True)
    def view(self):
        view_cls = import_string(self.import_name)
        return view_cls(*self.args, **self.kwargs)

   def __call__(self, *args, **kwargs):
       return self.view(*args, **kwargs)

    ...



Is there a reason you're not using the @cached_property decorator?

On Mon, Sep 19, 2011 at 2:35 PM, Armin Ronacher <armin.ronacher@active-4.com
> wrote:

> Hi,
>
> Why do do it like this?
>
>
> from threading import Lock
> from werkzeug import import_string
>
> class LazyView(object):
>
>    def __init__(self, import_name, *args, **kwargs):
>         self.__module__, self.__name__ = import_name.rsplit('.', 1)
>        self.import_name = import_name
>         self.args = args
>        self.kwargs = kwargs
>        self.lock = Lock()
>        self._view = None
>
>    def view(self):
>        if self._view is not None:
>            return self._view
>        with self.lock:
>            if self._view is None:
>                view_cls = import_string(self.import_name)
>                self._view = view_cls(*self.args, **self.kwargs)
>            return self._view
>
>    def __call__(self, *args, **kwargs):
>        return self.view(*args, **kwargs)
>
>
> view = LazyView('application.admin.FooView')
>
>
> Regards,
> Armin
>

Ignore the last line, I wrote that early in the email and forgot to remove
it after I gave the code more thought.

On Mon, Sep 19, 2011 at 3:44 PM, Sean Lynch <techniq35@gmail.com> wrote:

> It looks like this LazyView only works with class based views, so I would
> need to use the one on the patterns page for function based views (
> http://flask.pocoo.org/docs/patterns/lazyloading/)?
>
> Your implementation is much cleaner.  I didn't realize you could init the
> class instead of using as_view.  I guess that is way you have to use the
> thread lock, since you're not init'ing a new instance for each request, but
> using one instance as a callable.
>
> I was using the helper function to easily map urls:
>
> def url(rule, endpoint=None, import_name=None, **options):
>     view = LazyView('application.' + import_name)
>     app.add_url_rule(rule, endpoint, view, **options)
>
> So I guess I'll need to have 2 helper methods and use the appropriate
> LazyView implementation.
>
>
> As a side note, what do you think about extending @cached_property in
> Werkzeug to take in a "with_lock = True" parameter, so the LazyView could
> become:
>
> from werkzeug.utils import cached_property, import_string
>
> class LazyView(object):
>
>    def __init__(self, import_name, *args, **kwargs):
>        self.__module__, self.__name__ = import_name.rsplit('.', 1)
>        self.import_name = import_name
>        self.args = args
>        self.kwargs = kwargs
>
>     @cached_property(with_lock=True)
>     def view(self):
>         view_cls = import_string(self.import_name)
>         return view_cls(*self.args, **self.kwargs)
>
>    def __call__(self, *args, **kwargs):
>        return self.view(*args, **kwargs)
>
>     ...
>
>
>
> Is there a reason you're not using the @cached_property decorator?
>
>
> On Mon, Sep 19, 2011 at 2:35 PM, Armin Ronacher <
> armin.ronacher@active-4.com> wrote:
>
>> Hi,
>>
>> Why do do it like this?
>>
>>
>> from threading import Lock
>> from werkzeug import import_string
>>
>> class LazyView(object):
>>
>>    def __init__(self, import_name, *args, **kwargs):
>>         self.__module__, self.__name__ = import_name.rsplit('.', 1)
>>        self.import_name = import_name
>>         self.args = args
>>        self.kwargs = kwargs
>>        self.lock = Lock()
>>        self._view = None
>>
>>    def view(self):
>>        if self._view is not None:
>>            return self._view
>>        with self.lock:
>>            if self._view is None:
>>                view_cls = import_string(self.import_name)
>>                self._view = view_cls(*self.args, **self.kwargs)
>>            return self._view
>>
>>    def __call__(self, *args, **kwargs):
>>        return self.view(*args, **kwargs)
>>
>>
>> view = LazyView('application.admin.FooView')
>>
>>
>> Regards,
>> Armin
>>
>
>