mirror/nonsense
1
0
Fork 0
mirror of https://github.com/Findus23/nonsense.git synced 2024-09-19 16:03:50 +02:00
Code Releases Activity

update dependencies

Browse source
This commit is contained in:
Lukas Winkler 2019-12-20 18:31:42 +01:00
parent 56a0f87e80
commit b3929fb40d
Signed by: lukas
GPG key ID: 54DE4D798D244853
2 changed files with 26 additions and 30 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

10
fetch/fetch/spiders/nonsense.py
View file

@ -8,7 +8,7 @@ logger = logging.getLogger(__name__)
class NonsenseSpider(Spider):
name = "nonsense"
start_urls = [
"https://www.ikea.com/at/de/cat/produkte-functional/",
"https://www.ikea.com/at/de/cat/produkte-products/",
]
custom_settings = {
'FEED_FORMAT': 'json',
@ -16,13 +16,15 @@ class NonsenseSpider(Spider):
}
def parse(self, response):
products = response.css(".product-compact")
products = response.css("."
"")
for product in products:
name = product.css(".product-compact__name::text").extract_first()
description = product.css(".product-compact__type::text").extract_first().strip().strip(",")
yield {
'name': product.css(".product-compact__name::text").extract_first(),
'name': name,
'description': description,
}
for url in response.css("a.catalog-list__link::attr(href)"):
for url in response.css("a.range-catalog-list__link::attr(href)"):
if url is not None:
yield response.follow(url, callback=self.parse)

46
requirements.txt
View file

@ -1,38 +1,32 @@
asn1crypto==0.24.0
attrs==18.2.0
Automat==0.7.0
beautifulsoup4==4.7.1
certifi==2019.6.16
cffi==1.12.2
chardet==3.0.4
attrs==19.3.0
Automat==0.8.0
cffi==1.13.2
Click==7.0
constantly==15.1.0
cryptography==2.7
cssselect==1.0.3
cryptography==2.8
cssselect==1.1.0
Flask==1.1.1
hyperlink==18.0.0
hyperlink==19.0.0
idna==2.8
incremental==17.5.0
itsdangerous==1.1.0
Jinja2==2.10.1
lxml==4.3.4
Jinja2==2.10.3
lxml==4.4.2
MarkupSafe==1.1.1
parsel==1.5.1
pyasn1==0.4.4
pyasn1-modules==0.2.2
parsel==1.5.2
Protego==0.1.16
pyasn1==0.4.8
pyasn1-modules==0.2.7
pycparser==2.19
PyDispatcher==2.0.5
PyHamcrest==1.9.0
pyOpenSSL==19.0.0
PyYAML==5.1.1
pyOpenSSL==19.1.0
PyYAML==5.2
queuelib==1.5.0
requests==2.22.0
Scrapy==1.6.0
Scrapy==1.8.0
service-identity==18.1.0
six==1.11.0
soupsieve==1.9
Twisted==18.9.0
urllib3==1.25.3
w3lib==1.19.0
Werkzeug==0.15.4
zope.interface==4.6.0
six==1.13.0
Twisted==19.10.0
w3lib==1.21.0
Werkzeug==0.16.0
zope.interface==4.7.1