mirror of
https://github.com/Findus23/se-simulator.git
synced 2024-09-19 15:53:45 +02:00
replace all ' with "
This commit is contained in:
parent
4eca4fe9f8
commit
5fd7c55f49
4 changed files with 55 additions and 55 deletions
20
parsexml.py
20
parsexml.py
|
@ -9,21 +9,21 @@ def parse_posts(inputdir, outputdir):
|
||||||
skipped = 0
|
skipped = 0
|
||||||
iterator = etree.iterparse(inputdir + "/Posts.xml", events=("start", "end"))
|
iterator = etree.iterparse(inputdir + "/Posts.xml", events=("start", "end"))
|
||||||
_, root = next(iterator)
|
_, root = next(iterator)
|
||||||
with jsonlines.open(outputdir + '/Questions.jsonl', mode="w") as questions, \
|
with jsonlines.open(outputdir + "/Questions.jsonl", mode="w") as questions, \
|
||||||
jsonlines.open(outputdir + '/Answers.jsonl', mode="w") as answers, \
|
jsonlines.open(outputdir + "/Answers.jsonl", mode="w") as answers, \
|
||||||
jsonlines.open(outputdir + "/Titles.jsonl", "w") as titles:
|
jsonlines.open(outputdir + "/Titles.jsonl", "w") as titles:
|
||||||
for event, element in iterator:
|
for event, element in iterator:
|
||||||
title = element.get('Title')
|
title = element.get("Title")
|
||||||
# if element.get('Score') and int(element.get('Score')) <= 10:
|
# if element.get("Score") and int(element.get("Score")) <= 10:
|
||||||
# skipped += 1
|
# skipped += 1
|
||||||
# element.clear()
|
# element.clear()
|
||||||
# continue
|
# continue
|
||||||
if title:
|
if title:
|
||||||
titles.write(title)
|
titles.write(title)
|
||||||
body = element.get('Body')
|
body = element.get("Body")
|
||||||
if body:
|
if body:
|
||||||
text = html2text(body)
|
text = html2text(body)
|
||||||
if element.get('PostTypeId') == "1":
|
if element.get("PostTypeId") == "1":
|
||||||
questions.write(text)
|
questions.write(text)
|
||||||
else:
|
else:
|
||||||
answers.write(text)
|
answers.write(text)
|
||||||
|
@ -39,9 +39,9 @@ def parse_comments(inputdir, outputdir):
|
||||||
i = 0
|
i = 0
|
||||||
iterator = etree.iterparse(inputdir + "/Comments.xml", events=("start", "end"))
|
iterator = etree.iterparse(inputdir + "/Comments.xml", events=("start", "end"))
|
||||||
_, root = next(iterator)
|
_, root = next(iterator)
|
||||||
with jsonlines.open(outputdir + '/Comments.jsonl', mode="w") as comments:
|
with jsonlines.open(outputdir + "/Comments.jsonl", mode="w") as comments:
|
||||||
for event, element in iterator:
|
for event, element in iterator:
|
||||||
text = element.get('Text')
|
text = element.get("Text")
|
||||||
if text:
|
if text:
|
||||||
comments.write(text)
|
comments.write(text)
|
||||||
element.clear()
|
element.clear()
|
||||||
|
@ -56,9 +56,9 @@ def parse_usernames(inputdir, outputdir):
|
||||||
i = 0
|
i = 0
|
||||||
iterator = etree.iterparse(inputdir + "/Users.xml", events=("start", "end"))
|
iterator = etree.iterparse(inputdir + "/Users.xml", events=("start", "end"))
|
||||||
_, root = next(iterator)
|
_, root = next(iterator)
|
||||||
with jsonlines.open(outputdir + '/Usernames.jsonl', mode="w") as usernames:
|
with jsonlines.open(outputdir + "/Usernames.jsonl", mode="w") as usernames:
|
||||||
for event, element in iterator:
|
for event, element in iterator:
|
||||||
displayname = element.get('DisplayName')
|
displayname = element.get("DisplayName")
|
||||||
if displayname:
|
if displayname:
|
||||||
usernames.write(displayname)
|
usernames.write(displayname)
|
||||||
element.clear()
|
element.clear()
|
||||||
|
|
54
server.py
54
server.py
|
@ -42,11 +42,11 @@ question_count = utils.load_question_count()
|
||||||
|
|
||||||
@app.context_processor
|
@app.context_processor
|
||||||
def git_hash():
|
def git_hash():
|
||||||
return dict(git_hash=subprocess.check_output(['git', 'rev-parse', '--short', 'HEAD']).decode().strip())
|
return dict(git_hash=subprocess.check_output(["git", "rev-parse", "--short", "HEAD"]).decode().strip())
|
||||||
|
|
||||||
|
|
||||||
@app.route('/')
|
@app.route("/")
|
||||||
@app.route('/s/<string:site>')
|
@app.route("/s/<string:site>")
|
||||||
def index(site=None):
|
def index(site=None):
|
||||||
query = Question.select(Question, User, Site, Title, SQL(utils.rating_sql)).join(Site).switch(Question).join(
|
query = Question.select(Question, User, Site, Title, SQL(utils.rating_sql)).join(Site).switch(Question).join(
|
||||||
User).switch(
|
User).switch(
|
||||||
|
@ -62,7 +62,7 @@ def index(site=None):
|
||||||
paginated_query = PaginatedQuery(query, paginate_by=10, check_bounds=True)
|
paginated_query = PaginatedQuery(query, paginate_by=10, check_bounds=True)
|
||||||
pagearray = utils.create_pagination(paginated_query.get_page_count(), paginated_query.get_page())
|
pagearray = utils.create_pagination(paginated_query.get_page_count(), paginated_query.get_page())
|
||||||
return render_template(
|
return render_template(
|
||||||
'list.html',
|
"list.html",
|
||||||
pagearray=pagearray,
|
pagearray=pagearray,
|
||||||
num_pages=paginated_query.get_page_count(),
|
num_pages=paginated_query.get_page_count(),
|
||||||
page=paginated_query.get_page(),
|
page=paginated_query.get_page(),
|
||||||
|
@ -73,7 +73,7 @@ def index(site=None):
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
@app.route('/q/<string:slug>')
|
@app.route("/q/<string:slug>")
|
||||||
def question(slug):
|
def question(slug):
|
||||||
query = Question.select(Question, Title, User, Site) \
|
query = Question.select(Question, Title, User, Site) \
|
||||||
.join(Title).switch(Question) \
|
.join(Title).switch(Question) \
|
||||||
|
@ -92,12 +92,12 @@ def question(slug):
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
@app.route('/quiz/')
|
@app.route("/quiz/")
|
||||||
def hello():
|
def hello():
|
||||||
return redirect(url_for("quiz", difficulty="easy"), code=302)
|
return redirect(url_for("quiz", difficulty="easy"), code=302)
|
||||||
|
|
||||||
|
|
||||||
@app.route('/quiz/<string:difficulty>')
|
@app.route("/quiz/<string:difficulty>")
|
||||||
def quiz(difficulty):
|
def quiz(difficulty):
|
||||||
if difficulty not in ["easy", "hard"]:
|
if difficulty not in ["easy", "hard"]:
|
||||||
return abort(404)
|
return abort(404)
|
||||||
|
@ -124,7 +124,7 @@ def quiz(difficulty):
|
||||||
else:
|
else:
|
||||||
sites = None
|
sites = None
|
||||||
time2 = time.time()
|
time2 = time.time()
|
||||||
print('{} ms'.format((time2 - time1) * 1000.0))
|
print("{} ms".format((time2 - time1) * 1000.0))
|
||||||
return render_template(
|
return render_template(
|
||||||
"quiz.html",
|
"quiz.html",
|
||||||
question=question,
|
question=question,
|
||||||
|
@ -151,7 +151,7 @@ def quiz_api(id, guess, difficulty):
|
||||||
return jsonify({"site": model_to_dict(query)["site"], "correct": correct})
|
return jsonify({"site": model_to_dict(query)["site"], "correct": correct})
|
||||||
|
|
||||||
|
|
||||||
@app.route('/api/sites')
|
@app.route("/api/sites")
|
||||||
def sites():
|
def sites():
|
||||||
sites = Site.select().where(Site.last_download.is_null(False))
|
sites = Site.select().where(Site.last_download.is_null(False))
|
||||||
data = {}
|
data = {}
|
||||||
|
@ -160,8 +160,8 @@ def sites():
|
||||||
return jsonify(data)
|
return jsonify(data)
|
||||||
|
|
||||||
|
|
||||||
@app.route('/image')
|
@app.route("/image")
|
||||||
@app.route('/image/<int:site_id>')
|
@app.route("/image/<int:site_id>")
|
||||||
@limiter.limit("10 per minute")
|
@limiter.limit("10 per minute")
|
||||||
def image(site_id=None):
|
def image(site_id=None):
|
||||||
if site_id:
|
if site_id:
|
||||||
|
@ -172,15 +172,15 @@ def image(site_id=None):
|
||||||
pass
|
pass
|
||||||
|
|
||||||
site = DummySite()
|
site = DummySite()
|
||||||
site.foreground_color = 'black'
|
site.foreground_color = "black"
|
||||||
site.background_color = 'white'
|
site.background_color = "white"
|
||||||
# parameters
|
# parameters
|
||||||
text = "Stack Exchange\nSimulator"
|
text = "Stack Exchange\nSimulator"
|
||||||
selected_font = "/usr/share/fonts/truetype/dejavu/DejaVuSans.ttf"
|
selected_font = "/usr/share/fonts/truetype/dejavu/DejaVuSans.ttf"
|
||||||
font_size = 70
|
font_size = 70
|
||||||
W, H = (600, 600)
|
W, H = (600, 600)
|
||||||
# # get the size of the text
|
# # get the size of the text
|
||||||
img = Image.new('RGBA', (W, H), (site.background_color if site.background_color else "white"))
|
img = Image.new("RGBA", (W, H), (site.background_color if site.background_color else "white"))
|
||||||
font = ImageFont.truetype(selected_font, font_size)
|
font = ImageFont.truetype(selected_font, font_size)
|
||||||
draw = ImageDraw.Draw(img)
|
draw = ImageDraw.Draw(img)
|
||||||
w, h = draw.multiline_textsize(text, font)
|
w, h = draw.multiline_textsize(text, font)
|
||||||
|
@ -190,12 +190,12 @@ def image(site_id=None):
|
||||||
fill=(site.foreground_color if site.foreground_color else "black"))
|
fill=(site.foreground_color if site.foreground_color else "black"))
|
||||||
|
|
||||||
byte_io = BytesIO()
|
byte_io = BytesIO()
|
||||||
img.save(byte_io, 'PNG', optimize=True)
|
img.save(byte_io, "PNG", optimize=True)
|
||||||
byte_io.seek(0)
|
byte_io.seek(0)
|
||||||
return send_file(byte_io, mimetype='image/png')
|
return send_file(byte_io, mimetype="image/png")
|
||||||
|
|
||||||
|
|
||||||
@app.route('/api/vote/<string:type>/<int:id>/<string:vote>', methods=["POST"])
|
@app.route("/api/vote/<string:type>/<int:id>/<string:vote>", methods=["POST"])
|
||||||
@limiter.limit("10 per minute")
|
@limiter.limit("10 per minute")
|
||||||
def vote(type, id, vote):
|
def vote(type, id, vote):
|
||||||
if "voted" not in session:
|
if "voted" not in session:
|
||||||
|
@ -241,32 +241,32 @@ def ratelimit_handler(e):
|
||||||
return make_response(jsonify(error="access denied"), 403)
|
return make_response(jsonify(error="access denied"), 403)
|
||||||
|
|
||||||
|
|
||||||
if __name__ == '__main__':
|
if __name__ == "__main__":
|
||||||
import logging
|
import logging
|
||||||
|
|
||||||
logger = logging.getLogger('peewee')
|
logger = logging.getLogger("peewee")
|
||||||
logger.setLevel(logging.DEBUG)
|
logger.setLevel(logging.DEBUG)
|
||||||
logger.addHandler(logging.StreamHandler())
|
logger.addHandler(logging.StreamHandler())
|
||||||
|
|
||||||
|
|
||||||
@app.route('/static/js/<path:path>')
|
@app.route("/static/js/<path:path>")
|
||||||
def send_js(path):
|
def send_js(path):
|
||||||
return send_from_directory('web/static/js', path)
|
return send_from_directory("web/static/js", path)
|
||||||
|
|
||||||
|
|
||||||
app.debug = True
|
app.debug = True
|
||||||
app.wsgi_app = SassMiddleware(app.wsgi_app, manifests={
|
app.wsgi_app = SassMiddleware(app.wsgi_app, manifests={
|
||||||
'web': ('static/sass', 'static/css', '/static/css')
|
"web": ("static/sass", "static/css", "/static/css")
|
||||||
})
|
})
|
||||||
app.run()
|
app.run()
|
||||||
|
|
||||||
else:
|
else:
|
||||||
css, sourcemap = sass.compile(
|
css, sourcemap = sass.compile(
|
||||||
filename='web/static/sass/style.scss',
|
filename="web/static/sass/style.scss",
|
||||||
output_style='compressed',
|
output_style="compressed",
|
||||||
source_map_filename='web/static/css/style.css.map'
|
source_map_filename="web/static/css/style.css.map"
|
||||||
)
|
)
|
||||||
with open('web/static/css/style.css', 'w') as style_css:
|
with open("web/static/css/style.css", "w") as style_css:
|
||||||
style_css.write(css)
|
style_css.write(css)
|
||||||
with open('web/static/css/style.css.map', 'w') as style_css_map:
|
with open("web/static/css/style.css.map", "w") as style_css_map:
|
||||||
style_css_map.write(sourcemap)
|
style_css_map.write(sourcemap)
|
||||||
|
|
|
@ -21,7 +21,7 @@ def get_state_size(mode):
|
||||||
|
|
||||||
def load_chain(chainfile, mode):
|
def load_chain(chainfile, mode):
|
||||||
markov = get_markov(mode)
|
markov = get_markov(mode)
|
||||||
with open(chainfile, 'r') as myfile:
|
with open(chainfile, "r") as myfile:
|
||||||
data = myfile.read()
|
data = myfile.read()
|
||||||
print("using existing file\n")
|
print("using existing file\n")
|
||||||
return markov.from_json(data)
|
return markov.from_json(data)
|
||||||
|
@ -52,15 +52,15 @@ def generate_chain(sourcedir, chainfile, mode):
|
||||||
i += 1
|
i += 1
|
||||||
subtotal_chain = markovify.combine(chainlist)
|
subtotal_chain = markovify.combine(chainlist)
|
||||||
chain = markovify.combine([combined_cains, subtotal_chain])
|
chain = markovify.combine([combined_cains, subtotal_chain])
|
||||||
with open(chainfile, 'w') as outfile:
|
with open(chainfile, "w") as outfile:
|
||||||
outfile.write(chain.to_json())
|
outfile.write(chain.to_json())
|
||||||
print_ram()
|
print_ram()
|
||||||
return chain
|
return chain
|
||||||
|
|
||||||
|
|
||||||
def get_chain(url, mode):
|
def get_chain(url, mode):
|
||||||
sourcedir = 'raw/{url}'.format(url=url, type=mode)
|
sourcedir = "raw/{url}".format(url=url, type=mode)
|
||||||
chainfile = 'chains/{url}/{type}.chain.json'.format(url=url, type=mode)
|
chainfile = "chains/{url}/{type}.chain.json".format(url=url, type=mode)
|
||||||
if os.path.exists(chainfile):
|
if os.path.exists(chainfile):
|
||||||
return load_chain(chainfile, mode)
|
return load_chain(chainfile, mode)
|
||||||
else:
|
else:
|
||||||
|
|
28
utils.py
28
utils.py
|
@ -35,8 +35,8 @@ def get_files():
|
||||||
def file_hash(filename):
|
def file_hash(filename):
|
||||||
"""from https://stackoverflow.com/a/44873382/4398037"""
|
"""from https://stackoverflow.com/a/44873382/4398037"""
|
||||||
h = hashlib.sha1()
|
h = hashlib.sha1()
|
||||||
with open(filename, 'rb', buffering=0) as f:
|
with open(filename, "rb", buffering=0) as f:
|
||||||
for b in iter(lambda: f.read(128 * 1024), b''):
|
for b in iter(lambda: f.read(128 * 1024), b""):
|
||||||
h.update(b)
|
h.update(b)
|
||||||
return h.hexdigest()
|
return h.hexdigest()
|
||||||
|
|
||||||
|
@ -56,30 +56,30 @@ def get_settings(count):
|
||||||
|
|
||||||
|
|
||||||
def get_random_string(length):
|
def get_random_string(length):
|
||||||
return ''.join(random.choices(string.ascii_uppercase + string.digits, k=length))
|
return "".join(random.choices(string.ascii_uppercase + string.digits, k=length))
|
||||||
|
|
||||||
|
|
||||||
def prettydate(d):
|
def prettydate(d):
|
||||||
diff = datetime.now() - d
|
diff = datetime.now() - d
|
||||||
s = diff.seconds
|
s = diff.seconds
|
||||||
if diff.days > 7 or diff.days < 0:
|
if diff.days > 7 or diff.days < 0:
|
||||||
return d.strftime('%d %b %y')
|
return d.strftime("%d %b %y")
|
||||||
elif diff.days == 1:
|
elif diff.days == 1:
|
||||||
return '1 day ago'
|
return "1 day ago"
|
||||||
elif diff.days > 1:
|
elif diff.days > 1:
|
||||||
return '{} days ago'.format(diff.days)
|
return "{} days ago".format(diff.days)
|
||||||
elif s <= 1:
|
elif s <= 1:
|
||||||
return 'just now'
|
return "just now"
|
||||||
elif s < 60:
|
elif s < 60:
|
||||||
return '{} seconds ago'.format(s)
|
return "{} seconds ago".format(s)
|
||||||
elif s < 120:
|
elif s < 120:
|
||||||
return '1 minute ago'
|
return "1 minute ago"
|
||||||
elif s < 3600:
|
elif s < 3600:
|
||||||
return '{} minutes ago'.format(int(s / 60))
|
return "{} minutes ago".format(int(s / 60))
|
||||||
elif s < 7200:
|
elif s < 7200:
|
||||||
return '1 hour ago'
|
return "1 hour ago"
|
||||||
else:
|
else:
|
||||||
return '{} hours ago'.format(int(s / 3600))
|
return "{} hours ago".format(int(s / 3600))
|
||||||
|
|
||||||
|
|
||||||
def create_pagination(num_pages, page, padding=2):
|
def create_pagination(num_pages, page, padding=2):
|
||||||
|
@ -135,10 +135,10 @@ def is_light_color(hex):
|
||||||
|
|
||||||
|
|
||||||
def save_question_count(count):
|
def save_question_count(count):
|
||||||
with open('count.txt', 'w') as f:
|
with open("count.txt", "w") as f:
|
||||||
f.write(str(count))
|
f.write(str(count))
|
||||||
|
|
||||||
|
|
||||||
def load_question_count():
|
def load_question_count():
|
||||||
with open('count.txt', 'r') as f:
|
with open("count.txt", "r") as f:
|
||||||
return int(f.readline())
|
return int(f.readline())
|
||||||
|
|
Loading…
Reference in a new issue