mirror of
https://github.com/Findus23/acronomy.git
synced 2024-09-19 15:33:45 +02:00
use custom user agent
This commit is contained in:
parent
d53bb1ebe4
commit
2c3a42c077
4 changed files with 21 additions and 7 deletions
|
@ -6,6 +6,8 @@ from PIL import Image
|
|||
from django.core.files.base import ContentFile
|
||||
from django.db import models
|
||||
|
||||
from acros.utils.apis import requests_session
|
||||
|
||||
|
||||
class Host(models.Model):
|
||||
host = models.CharField(max_length=100)
|
||||
|
@ -21,7 +23,7 @@ class Host(models.Model):
|
|||
def save(self, *args, **kwargs):
|
||||
if not self.fetched or True:
|
||||
with TemporaryFile("rb+") as fd:
|
||||
r = requests.get(f"https://external-content.duckduckgo.com/ip3/{self.host}.ico")
|
||||
r = requests_session.get(f"https://external-content.duckduckgo.com/ip3/{self.host}.ico")
|
||||
if r.ok:
|
||||
filename = self.host + ".png"
|
||||
for chunk in r.iter_content(chunk_size=128):
|
||||
|
|
|
@ -4,7 +4,7 @@ import requests
|
|||
from django.core.files import File
|
||||
from django.db import models
|
||||
|
||||
from acros.utils.apis import WikipediaImageAPIObject, NotFoundError
|
||||
from acros.utils.apis import WikipediaImageAPIObject, NotFoundError, requests_session
|
||||
|
||||
|
||||
class WikipediaImage(models.Model):
|
||||
|
@ -36,7 +36,7 @@ class WikipediaImage(models.Model):
|
|||
if filename.endswith(".svg"):
|
||||
filename += ".png"
|
||||
with TemporaryFile("rb+") as fd:
|
||||
r = requests.get(img.thumburl)
|
||||
r = requests_session.get(img.thumburl)
|
||||
for chunk in r.iter_content(chunk_size=128):
|
||||
fd.write(chunk)
|
||||
image_file = File(fd)
|
||||
|
|
|
@ -2,9 +2,20 @@ from typing import Tuple, Optional
|
|||
|
||||
import requests
|
||||
from bs4 import BeautifulSoup
|
||||
from django.core.cache import cache
|
||||
|
||||
from acros.utils.html import clean_html, string_to_bool
|
||||
|
||||
requests_session = requests.Session()
|
||||
commit = cache.get("commit")
|
||||
if commit:
|
||||
commit_version = commit[:6]
|
||||
else:
|
||||
commit_version = ""
|
||||
requests_session.headers.update({
|
||||
"User-Agent": f"Acronomy {commit_version} (https://acronomy.lw1.at)"
|
||||
})
|
||||
|
||||
|
||||
class NotFoundError(FileNotFoundError):
|
||||
"""Request could not be found in API"""
|
||||
|
@ -15,7 +26,7 @@ class WikipediaAPISummary:
|
|||
urlbase = "https://en.wikipedia.org/api/rest_v1/page/summary/"
|
||||
|
||||
def __init__(self, title: str):
|
||||
r = requests.get(self.urlbase + title.replace("/", "%2F"))
|
||||
r = requests_session.get(self.urlbase + title.replace("/", "%2F"))
|
||||
try:
|
||||
r.raise_for_status()
|
||||
except requests.HTTPError:
|
||||
|
@ -50,7 +61,7 @@ class WikipediaAPISummary:
|
|||
|
||||
|
||||
def get_website_title(url: str) -> str:
|
||||
r = requests.get(url)
|
||||
r = requests_session.get(url)
|
||||
r.raise_for_status()
|
||||
soup = BeautifulSoup(r.text, features="html.parser")
|
||||
title = soup.find("title")
|
||||
|
@ -61,7 +72,7 @@ class WikipediaImageAPIObject:
|
|||
def __init__(self, filename: str):
|
||||
self.filename = filename
|
||||
print(self.api_url)
|
||||
r = requests.get(self.api_url)
|
||||
r = requests_session.get(self.api_url)
|
||||
r.raise_for_status()
|
||||
self.data = r.json()
|
||||
self.image_obj = list(self.data["query"]["pages"].values())[0]
|
||||
|
|
|
@ -3,6 +3,7 @@ from datetime import date
|
|||
import requests
|
||||
|
||||
from acronomy.settings import MATOMO_API_KEY
|
||||
from acros.utils.apis import requests_session
|
||||
|
||||
|
||||
def fetch_matomo_pages():
|
||||
|
@ -16,5 +17,5 @@ def fetch_matomo_pages():
|
|||
"&method=Actions.getPageUrls" \
|
||||
"&module=API" \
|
||||
f"&token_auth={MATOMO_API_KEY}"
|
||||
r = requests.get(url)
|
||||
r = requests_session.get(url)
|
||||
return r.json()
|
||||
|
|
Loading…
Reference in a new issue