Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Usage of a lambda function as the request callback #13

Open
wants to merge 3 commits into
base: master
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -10,5 +10,6 @@ A Flake8 plugin to catch common issues on Scrapy spiders.
| SCP02 | There are URLs in `allowed_domains` |
| SCP03 | Usage of `urljoin(response.url, '/foo')` instead of `response.urljoin('/foo')` |
| SCP04 | Usage of `Selector(response)` in callback |
| SCP05 | Usage of a lambda function as the request callback |

This is a work in progress, so new issues will be added to this list.
40 changes: 40 additions & 0 deletions finders/unsupported.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,40 @@
import ast
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@stummjr I'm wondering about the name of this file unsupported.py, is using lambda something that's unsupported on Scrapy?

Copy link
Owner Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@rocioar that's right, request serialization issues would happen if there as a lambda function passed as the callback. I have to confirm if that's still an issue, though. :)


from finders import IssueFinder


class LambdaCallbackIssueFinder(IssueFinder):
msg_code = 'SCP05'
msg_info = 'callback should not be a lambda'

def is_scrapy_request(self, node):
return (
isinstance(node.func, ast.Attribute) and
node.func.value.id == 'scrapy' and
node.func.attr == 'Request'
)

def is_raw_request(self, node):
return (
isinstance(node.func, ast.Name) and
node.func.id == 'Request'
)

def issue_applies(self, node):
return (
self.is_raw_request(node) or
self.is_scrapy_request(node)
)

def find_issues(self, node):
if not self.issue_applies(node):
return

if len(node.args) >= 2:
callback = node.args[1]
if isinstance(callback, ast.Lambda):
return [(callback.lineno, callback.col_offset, self.message)]

for kw in node.keywords:
if kw.arg == 'callback' and isinstance(kw.value, ast.Lambda):
return [(node.lineno, node.col_offset, self.message)]
3 changes: 2 additions & 1 deletion flake8_scrapy.py
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@
UnreachableDomainIssueFinder, UrlInAllowedDomainsIssueFinder,
)
from finders.oldstyle import OldSelectorIssueFinder, UrlJoinIssueFinder

from finders.unsupported import LambdaCallbackIssueFinder

__version__ = '0.0.1'

Expand All @@ -22,6 +22,7 @@ def __init__(self, *args, **kwargs):
],
'Call': [
UrlJoinIssueFinder(),
LambdaCallbackIssueFinder(),
]
}

Expand Down
16 changes: 16 additions & 0 deletions tests/test_unsupported.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,16 @@
import pytest

from . import run_checker


@pytest.mark.parametrize('code,expected', [
('Request(x, callback=lambda x: x)', 1),
('scrapy.Request(x, callback=lambda x: x)', 1),
('scrapy.Request(x, lambda x: x)', 1),
('Request(x, callback=self.parse)', 0),
('scrapy.Request(x, callback=self.parse)', 0),
('scrapy.Request(x, self.parse)', 0),
])
def test_find_lambda_callback_issue(code, expected):
issues = run_checker(code)
assert len(issues) == expected