Merge pull request #38 from Findus23/exists

check if article exists in wallabag before adding

main.py

@@ -7,7 +7,6 @@ from urllib.parse import urljoin
 import aiohttp
 import feedparser
 import yaml
-from raven import Client
 from wallabag_api.wallabag import Wallabag
 
 import github_stars
@@ -42,14 +41,6 @@ with open("sites.yaml", 'r') as stream:
         sites = None
         exit(1)
 
-if "sentry_url" in config and ("debug" not in config or not config["debug"]):
-    client = Client(
-        dsn=config["sentry_url"],
-        processors=(
-            'raven.processors.SanitizePasswordsProcessor',
-        )
-    )
-
 
 async def fetch(session, url):
     try:
@@ -57,8 +48,6 @@ async def fetch(session, url):
             return await response.text()
     except Exception as e:
         logging.exception("failed to fetch {url}".format(url=url))
-        if 'client' in locals():
-            client.captureException(data={url:url})
 
 
 async def main(loop, sites):
@@ -101,8 +90,12 @@ async def handle_feed(session, wall, sitetitle, site):
                 title = sitetitle + ": " + article.title
             else:
                 title = article.title
+            url = urljoin(site["url"], article.link)
+            exists = await wall.entries_exists(url)
+            if exists["exists"]:
+                logger.info("already found in wallabag: " + article.title)
             if "debug" not in config or not config["debug"]:
-                await wall.post_entries(url=urljoin(site["url"], article.link), title=title, tags=tags)
+                await wall.post_entries(url=url, title=title, tags=tags)
     else:
         logger.debug("no latest_article: " + sitetitle)
     if f.entries:

requirements.txt

@@ -1,5 +1,5 @@
 aiodns==1.1.1
-aiohttp==3.1.2
+aiohttp==3.1.3
 async-timeout==2.0.1
 attrs==17.4.0
 certifi==2018.1.18
@@ -7,11 +7,10 @@ chardet==3.0.4
 feedparser==5.2.1
 idna==2.6
 idna-ssl==1.0.1
-multidict==4.1.0
+multidict==4.2.0
 pycares==2.3.0
 PyYAML==3.12
-raven==6.6.0
 requests==2.18.4
 urllib3==1.22
-wallabag-api==1.2.0
+wallabag-api==1.2.2
 yarl==1.1.1