diff --git a/consume.py b/consume.py index c968736..9c96343 100644 --- a/consume.py +++ b/consume.py @@ -9,7 +9,9 @@ from parsexml import parse_posts, parse_comments, parse_usernames from utils import * files = get_files() -# os.chdir("/mydir") + +# TODO: name sites/id after real url + for file in glob.glob("downloads/**/*.7z"): if "meta" in file: continue diff --git a/models.py b/models.py index 9fbb8e5..170ac59 100644 --- a/models.py +++ b/models.py @@ -39,6 +39,7 @@ class Question(BaseModel): user = ForeignKeyField(User) site = ForeignKeyField(Site) datetime = DateTimeField() + random = IntegerField() class Answer(BaseModel): diff --git a/requirements.txt b/requirements.txt index c4e128b..469585e 100644 --- a/requirements.txt +++ b/requirements.txt @@ -3,18 +3,30 @@ backports.csv==1.0.5 beautifulsoup4==4.6.0 certifi==2018.1.18 chardet==3.0.4 +click==6.7 clint==0.5.1 docopt==0.6.2 +Flask==0.13.dev0 +gunicorn==19.7.1 html2text==2018.1.9 idna==2.6 internetarchive==1.7.7 +itsdangerous==0.24 +Jinja2==2.10 jsonlines==1.2.0 jsonpatch==1.21 jsonpointer==2.0 +libsass==0.14.2 lxml==4.2.0 +markovify==0.7.1 +MarkupSafe==1.0 mysqlclient==1.3.12 -peewee==3.1.3 +nltk==3.2.5 +peewee==3.1.5 +python-slugify==1.2.4 requests==2.18.4 schema==0.6.7 six==1.11.0 +Unidecode==1.0.22 urllib3==1.22 +Werkzeug==0.14.1 diff --git a/server.py b/server.py index 3888e20..2f27a1a 100644 --- a/server.py +++ b/server.py @@ -1,7 +1,6 @@ -from datetime import datetime - -from flask import render_template, flash -from playhouse.shortcuts import model_to_dict +import sass +from flask import render_template +from playhouse.flask_utils import PaginatedQuery from sassutils.wsgi import SassMiddleware import utils @@ -10,45 +9,27 @@ from models import * app.jinja_env.globals.update(prettydate=utils.prettydate) -def query_to_response(query, limit=10, key=False, sort=False, offset=None, list=None, **kwargs): - """ - - :param sort: boolean - :param offset: int - :type key: str - :type limit: int|boolean - :param **kwargs - :type query: peewee.ModelSelect - """ - if limit: - query = query.limit(limit) - print(query.sql()) - data = {} if key is not False else [] - order = int(offset) if offset else 0 - for i in query: - element = model_to_dict(i, **kwargs) - if list: - element = element[list] - if sort: - element["order"] = order - order += 1 - if key is not False: - if "." in key: - key1, key2 = key.split(".") - data[getattr(getattr(i, key1), key2)] = element - else: - data[getattr(i, key)] = element - else: - data.append(element) - return data - @app.route('/') def index(): - query = Question.select().limit(10) - # return query_to_response(Question.select().limit(10), limit=False, max_depth=1) - # return query_to_response(query, max_depth=1) - return render_template('list.html', questions=query_to_response(query, max_depth=1)) + select = """ + *, + ((upvotes + 1.9208) / (upvotes + downvotes) - + 1.96 * SQRT((upvotes * downvotes) / (upvotes + downvotes) + 0.9604) / + (upvotes + downvotes)) / (1 + 3.8416 / (upvotes + downvotes)) + AS ci_lower_bound + """ + query = Question.select(SQL(select)).order_by(SQL("ci_lower_bound DESC, random")) + print(query.sql()) + paginated_query = PaginatedQuery(query, paginate_by=10, check_bounds=True) + pagearray = utils.create_pagination(paginated_query.get_page_count(), paginated_query.get_page()) + return render_template( + 'list.html', + pagearray=pagearray, + num_pages=paginated_query.get_page_count(), + page=paginated_query.get_page(), + questions=paginated_query.get_object_list() + ) @app.route('/q/') @@ -66,3 +47,5 @@ if __name__ == '__main__': 'web': ('static/sass', 'static/css', '/static/css') }) app.run() +else: + sass.compile(dirname=('web/static/sass', 'web/static/css'), output_style='compressed') diff --git a/templates/base.html b/templates/base.html index 7b7ff1e..453f2cc 100644 --- a/templates/base.html +++ b/templates/base.html @@ -1,6 +1,10 @@ Flaskr - +{% if config['DEBUG'] %} + +{% else %} + +{% endif %}

Flaskr

{% for message in get_flashed_messages() %} diff --git a/templates/list.html b/templates/list.html index c0ea81d..d454caf 100644 --- a/templates/list.html +++ b/templates/list.html @@ -1,5 +1,7 @@ {% extends "base.html" %} +{% from 'macros.html' import pagination %} {% block body %} + {{ pagination(pagearray, num_pages, page, True) }} {% for question in questions %}
@@ -18,12 +20,9 @@
{{ question.text }}
- asked {{ prettydate(question.datetime) }} by {{ question.user.username }} + asked {{ prettydate(question.datetime) }} by {{ question.user.username }}
- {% else %} - Unbelievable. No entries here so far {% endfor %} - 
{{ questions[0]|pprint }}
{% endblock %} diff --git a/templates/macros.html b/templates/macros.html new file mode 100644 index 0000000..0d30637 --- /dev/null +++ b/templates/macros.html @@ -0,0 +1,44 @@ +{% macro printpage(i,active) %} + {# {{ i }}#} + {{ i }} +{% endmacro %} + +{% macro pagination(pagearray, num_pages, page,top) %} + + +{%- endmacro %} + diff --git a/todb.py b/todb.py index 66e46e1..1fa10d2 100644 --- a/todb.py +++ b/todb.py @@ -2,6 +2,7 @@ from datetime import datetime from slugify import slugify +import utils from models import * from text_generator import get_chain, generate_text @@ -37,12 +38,12 @@ def add_question(site, count=100): user = users[i] print(user.username) time = datetime.now() - Question.create(text=text, title_id=title, user_id=user, site_id=site, datetime=time) + Question.create(text=text, title_id=title, user_id=user, site_id=site, datetime=time, random=utils.rand()) if __name__ == "__main__": - query = Site.select().where(Site.last_download.is_null(False)).limit(1) - s = query.get() - add_username(s) - add_title(s) - add_question(s) + query = Site.select().where(Site.last_download.is_null(False)) + for s in query: + add_username(s) + add_title(s) + add_question(s) diff --git a/utils.py b/utils.py index 5ef36e8..a64bb93 100644 --- a/utils.py +++ b/utils.py @@ -62,7 +62,6 @@ def get_random_string(length): def prettydate(d): diff = datetime.now() - d s = diff.seconds - print(diff) if diff.days > 7 or diff.days < 0: return d.strftime('%d %b %y') elif diff.days == 1: @@ -81,3 +80,22 @@ def prettydate(d): return '1 hour ago' else: return '{} hours ago'.format(int(s / 3600)) + + +def create_pagination(num_pages, page, padding=2): + pages = ["1"] + i = 2 + while i <= num_pages: + if i < (page - padding - 1): + pages.append("d") + i = page - padding + elif (i > (page + padding)) and (num_pages > (page + padding + 2)): + pages.append("d") + i = num_pages + pages.append(str(i)) + i += 1 + return pages + + +def rand(): + return random.randint(-2**31, 2**31-1) diff --git a/web/static/sass/_pagination.scss b/web/static/sass/_pagination.scss new file mode 100644 index 0000000..a7f035b --- /dev/null +++ b/web/static/sass/_pagination.scss @@ -0,0 +1,52 @@ +.pagination { + &.top {} + &.bottom { + text-align: right; + + @media (max-width: $mobile-width) { + text-align: center; + } + } + + a { + .mobile { + display: none; + } + + font-size: 18px; + display: inline-block; + padding: 8px 20px; + margin-left: -5px; + + svg { + height: 18px; + vertical-align: middle; + } + + @media (min-width: 500px) { + &:hover:not(.disabled), &.active { + color: $primary-dark; + border-bottom: 3px solid; + } + } + + &.disabled { + color: $lightTextGray; + + svg { + fill: $lightTextGray; + } + } + + @media (max-width: $mobile-width) { + &.active::before { + content: "Seite "; + } + + &.other { + display: none; + } + } + } +} + diff --git a/web/static/sass/variables.scss b/web/static/sass/_variables.scss similarity index 100% rename from web/static/sass/variables.scss rename to web/static/sass/_variables.scss diff --git a/web/static/sass/style.scss b/web/static/sass/style.scss index f1ae270..aeefc74 100644 --- a/web/static/sass/style.scss +++ b/web/static/sass/style.scss @@ -3,6 +3,8 @@ $link-hover-color: #3af; @import "../../milligram/src/milligram"; +@import "pagination"; + body { font-family: Arial, "Helvetica Neue", Helvetica, sans-serif; color: #111;