add stats endpoint

2024-09-19 15:23:44 +02:00 · 2021-10-26 21:03:45 +02:00 · 2021-10-26 21:03:45 +02:00 · 5bb7d7081a
commit 5bb7d7081a
parent 217d0d389f
9 changed files with 283 additions and 120 deletions
--- a/app.py
+++ b/app.py
@ -28,4 +28,4 @@ app.config.from_object(__name__)
 cache = Cache(app)
 flask_db = FlaskDB(app)

-db = flask_db.database
+db: PooledPostgresqlDatabase = flask_db.database
--- a/poetry.lock
+++ b/poetry.lock
@ -260,6 +260,20 @@ python-versions = "*"
 cymem = ">=2.0.2,<2.1.0"
 murmurhash = ">=0.28.0,<1.1.0"

+[[package]]
+name = "prettytable"
+version = "2.2.1"
+description = "A simple Python library for easily displaying tabular data in a visually appealing ASCII table format"
+category = "main"
+optional = false
+python-versions = ">=3.6"
+
+[package.dependencies]
+wcwidth = "*"
+
+[package.extras]
+tests = ["pytest", "pytest-cov"]
+
 [[package]]
 name = "psycopg2"
 version = "2.9.1"
@ -552,6 +566,14 @@ category = "main"
 optional = false
 python-versions = "*"

+[[package]]
+name = "wcwidth"
+version = "0.2.5"
+description = "Measures the displayed width of unicode strings in a terminal"
+category = "main"
+optional = false
+python-versions = "*"
+
 [[package]]
 name = "werkzeug"
 version = "2.0.2"
@ -574,7 +596,7 @@ python-versions = "*"
 [metadata]
 lock-version = "1.1"
 python-versions = "^3.8"
-content-hash = "6a4064e9d63518ddd6d84e41e6741f37019ab84ecdd725df6442e0d18713ede4"
+content-hash = "56fd823cbab8434d8c64b77fa42718dc4344dff8fed89f601adf42159ea95773"

 [metadata.files]
 about-time = [
@ -784,6 +806,10 @@ preshed = [
    {file = "preshed-3.0.6-cp39-cp39-win_amd64.whl", hash = "sha256:92a8f49d17a63537a8beed48a049b62ef168ca07e0042a5b2bcdf178a1fb5d48"},
    {file = "preshed-3.0.6.tar.gz", hash = "sha256:fb3b7588a3a0f2f2f1bf3fe403361b2b031212b73a37025aea1df7215af3772a"},
 ]
+prettytable = [
+    {file = "prettytable-2.2.1-py3-none-any.whl", hash = "sha256:09fb2c7f93e4f93e0235f05ae199ac3f16da3a251b2cfa1c7108b34ede298fa3"},
+    {file = "prettytable-2.2.1.tar.gz", hash = "sha256:6d465005573a5c058d4ca343449a5b28c21252b86afcdfa168cdc6a440f0b24c"},
+]
 psycopg2 = [
    {file = "psycopg2-2.9.1-cp36-cp36m-win32.whl", hash = "sha256:7f91312f065df517187134cce8e395ab37f5b601a42446bdc0f0d51773621854"},
    {file = "psycopg2-2.9.1-cp36-cp36m-win_amd64.whl", hash = "sha256:830c8e8dddab6b6716a4bf73a09910c7954a92f40cf1d1e702fb93c8a919cc56"},
@ -938,6 +964,10 @@ wasabi = [
    {file = "wasabi-0.8.2-py3-none-any.whl", hash = "sha256:a493e09d86109ec6d9e70d040472f9facc44634d4ae6327182f94091ca73a490"},
    {file = "wasabi-0.8.2.tar.gz", hash = "sha256:b4a36aaa9ca3a151f0c558f269d442afbb3526f0160fd541acd8a0d5e5712054"},
 ]
+wcwidth = [
+    {file = "wcwidth-0.2.5-py2.py3-none-any.whl", hash = "sha256:beb4802a9cebb9144e99086eff703a642a13d6a0052920003a230f3294bbe784"},
+    {file = "wcwidth-0.2.5.tar.gz", hash = "sha256:c4d647b99872929fdb7bdcaa4fbe7f01413ed3d98077df798530e5b04f116c83"},
+]
 werkzeug = [
    {file = "Werkzeug-2.0.2-py3-none-any.whl", hash = "sha256:63d3dc1cf60e7b7e35e97fa9861f7397283b75d765afcaefd993d6046899de8f"},
    {file = "Werkzeug-2.0.2.tar.gz", hash = "sha256:aa2bb6fc8dee8d6c504c0ac1e7f5f7dc5810a9903e793b6f715a9f015bdadb9a"},
--- a/pyproject.toml
+++ b/pyproject.toml
@ -20,6 +20,7 @@ sentry-sdk = {extras = ["flask"], version = "^1.0.0"}
 blinker = "^1.4" # tmp workaround to make flask-sentry work
 Flask-Caching = "^1.10.1"
 redis = "^3.5.3"
+prettytable = "^2.2.1"

 [tool.poetry.dev-dependencies]

--- a/server.py
+++ b/server.py
@ -12,6 +12,7 @@ from models import *
 # logger = logging.getLogger('peewee')
 # logger.addHandler(logging.StreamHandler())
 # logger.setLevel(logging.DEBUG)
+from stats import TotalWords, MostCommonNounChunks, LongestNounChunks, LinesPerPerson
 from suggestions import suggestions


@ -241,6 +242,29 @@ def transcript():
    })


+@app.route("/api/stats")
+@cache.cached(timeout=60 * 60 * 24)
+def stats():
+    return jsonify({
+        "TotalWords": TotalWords().as_data(),
+        "MostCommonNounChunks": MostCommonNounChunks().as_data(),
+        "LongestNounChunks": LongestNounChunks().as_data(),
+        "LinesPerPerson": LinesPerPerson().as_data()
+    })
+
+
+@app.route("/api/stats/text")
+@cache.cached(timeout=60 * 60 * 24)
+def stats_text():
+    text = ""
+
+    for stats_class in [TotalWords, MostCommonNounChunks, LongestNounChunks, LinesPerPerson]:
+        text += type(stats_class()).__name__.center(100, "#") + "\n"
+        text += stats_class().as_plaintext() + "\n\n"
+
+    return Response(text, mimetype='text/plain')
+
+
 if __name__ == "__main__":
    import logging

--- a/stats.py
+++ b/stats.py
@ -0,0 +1,86 @@
+from abc import ABC, abstractmethod
+from typing import List, Dict, Any
+
+from prettytable import PrettyTable
+from psycopg2._psycopg import cursor
+
+from app import db
+
+
+class Stats(ABC):
+    query: str
+
+    def execute(self) -> cursor:
+        return db.execute_sql(self.query)
+
+    @abstractmethod
+    def as_data(self):
+        ...
+
+    def as_plaintext(self) -> str:
+        cur = self.execute()
+        x = PrettyTable()
+        x.add_rows(cur.fetchall())
+        x.field_names = [d.name for d in cur.description]
+        return x.get_string()
+
+
+class MultiColumnStats(Stats):
+
+    def as_data(self) -> List[Dict[str, Any]]:
+        data = []
+        cur = self.execute()
+        column_names = [d.name for d in cur.description]
+        for row in cur.fetchall():
+            single_stat = {}
+            for i, value in enumerate(row):
+                single_stat[column_names[i]] = value
+            data.append(single_stat)
+        return data
+
+
+class SingleValueStats(Stats):
+
+    def as_data(self) -> int:
+        result = self.execute().fetchone()
+        return result[-1]
+
+
+class LinesPerPerson(MultiColumnStats):
+    query = """
+select name, count(name) as count, sum(length(text)) as count_chars
+from line
+         join person p on line.person_id = p.id
+group by name
+order by count_chars desc;
+"""
+
+
+class MostCommonNounChunks(MultiColumnStats):
+    query = """
+select text, sum(count) as num_occurrence
+from phrase
+group by text
+order by num_occurrence desc
+limit 1000;
+"""
+
+
+class LongestNounChunks(MultiColumnStats):
+    query = """
+select text, char_length(phrase.text) as length
+from phrase
+order by length desc
+limit 100;
+;
+"""
+
+
+class TotalWords(SingleValueStats):
+    query = """
+select sum(array_length(regexp_split_to_array(text,'\\s'),1)) from line
+"""
+
+
+if __name__ == '__main__':
+    print(TotalWords().as_data())
--- a/tests.sql
+++ b/tests.sql
@ -1,22 +1,3 @@
-- number of subtitle lines per person
-select name, count(name) as count, sum(length(text)) as chars
-from line
-         join person p on line.person_id = p.id
-group by name
-order by chars desc;
-
-- most common noun chunks
-select text, sum(count) as count
-from phrase
-group by text
-order by count desc;
-
-- longest noun chunks
-select text, char_length(phrase.text) as len
-from phrase
-order by len desc;
-
-
 select e.pretty_title, text,char_length(line.text) as len from line join episode e on e.id = line.episode_id order by len desc;

 delete
--- a/web/src/App.vue
+++ b/web/src/App.vue
@ -1,8 +1,49 @@
 <template>
    <div id="app">
        <div class="container">
-            <router-view/>
+            <div id="contentwrapper">
+                <transition name="fade">
+                    <div id="page-mask" v-if="showIntro"></div>
+                </transition>
+
+                <Intro v-if="showIntro"></Intro>
+
+                <router-view/>
+                <footer>
+                    <button @click="showIntro=true" class="btn btn-link">About this website</button>
+                    <router-link :to="{name:'episodes'}">Episode overview</router-link>
+                    <a href="https://lw1.at">My other Projects</a>
+                    <a href="https://lw1.at/i">Privacy Policy</a>
+                </footer>
+            </div>
        </div>
    </div>
 </template>

+<script lang="ts">
+import Vue from "vue";
+import Intro from "@/components/Intro.vue";
+
+export default Vue.extend({
+  name: "App",
+  components: {
+    Intro,
+  },
+  mounted(): void {
+    if (localStorage.getItem("showIntro") !== null) {
+      this.showIntro = localStorage.getItem("showIntro") === "true";
+    }
+  },
+  data() {
+    return {
+      showIntro: true,
+    };
+  },
+  watch: {
+    showIntro(value: boolean): void {
+      localStorage.setItem("showIntro", value.toString());
+    }
+  },
+
+});
+</script>
--- a/web/src/views/Episodes.vue
+++ b/web/src/views/Episodes.vue
@ -1,5 +1,5 @@
 <template>
-    <div id="contentwrapper" class="text-page">
+    <div class="text-page">
        <h1>Episode Overview</h1>
        <div v-for="series in series_data" :key="series.meta.id" class="episode-table">
            <h2 :id="seriesID(series.meta)">{{ series.meta.title }}</h2>
@ -7,6 +7,7 @@
                <thead>
                <tr>
                    <th>Title</th>
+                    <th></th>
                    <th>Episode</th>
                    <th>Video</th>
                    <th>Upload Date</th>
@ -21,9 +22,12 @@
                        {{ episode.pretty_title }}
                    </a>
                    </td>
+                    <td>
+                        <router-link class="btn" :to="transcriptLink(episode,series.meta)">T</router-link>
+                    </td>
                    <td>{{ episode.episode_number }}</td>
                    <td>{{ episode.video_number }}</td>
-                    <td>{{episode.upload_date}}</td>
+                    <td>{{ episode.upload_date }}</td>
                    <td class="text-center">
                        <CheckMark :status="episode.downloaded"></CheckMark>
                    </td>
@ -45,6 +49,7 @@ import Vue from "vue";
 import {EpisodeDetailed, Series, SeriesData} from "@/interfaces";
 import {baseURL} from "@/utils";
 import CheckMark from "@/components/CheckMark.vue";
+import {Location} from "vue-router";

 export default Vue.extend({
  name: "Episodes",
@ -72,6 +77,16 @@ export default Vue.extend({
    },
    seriesID(series: Series, withHash: boolean) {
      return (withHash ? "#" : "") + `series-${series.id}`;
+    },
+    transcriptLink(episode: EpisodeDetailed, series: Series): Location {
+      console.log(series.slug)
+      return {
+        name: "transcript",
+        params: {
+          episodeNr: episode.episode_number.toString(),
+          series: series.slug
+        }
+      };
    }
  }
 });
--- a/web/src/views/Home.vue
+++ b/web/src/views/Home.vue
@ -1,98 +1,90 @@
 <template>
-    <div id="contentwrapper">
-        <div class="home">
-            <div class="title">
-                <h1>Critical Role Search</h1>
-                <span>Find your favourite Critical Role quote!</span>
-            </div>
-            <transition name="fade">
-                <div id="page-mask" v-if="showIntro || showYtOptIn|| showSeriesSelector"></div>
-            </transition>
-            <SeriesSelector v-if="showSeriesSelector" :serverData="serverData"></SeriesSelector>
-            <Intro v-if="showIntro"></Intro>
-            <div v-if="showYtOptIn" class="ytoptin popup">
-                <div>
-                    <p>This play button allows you to watch the exact timestamp of this quote on YouTube.
-                        This means that the YouTube video is loaded on this page and data is sent to YouTube/Google.
-                        (<a href="https://lw1.at/i">Privacy Policy</a>)
-                    </p>
-                    <p>
-                        If you expected this to happen, simply continue and you won't be asked again. Otherwise you can
-                        abort or watch this timestamp directly on YouTube.
-                    </p>
-                </div>
-                <div class="buttonrow">
-                    <button class="btn" @click="showYtOptIn=false">abort</button>
-                    <a class="btn" :href="ytLink" target="youtube" rel="noopener" @click="showYtOptIn=false">YouTube</a>
-                    <button class="btn" @click="doYtOptIn">continue</button>
-                </div>
-            </div>
-            <div v-if="showYT" class="ytwrapper">
-                <button class="btn" @click="closeVideo">Hide</button>
-                <youtube :nocookie="true" ref="youtube" @ready="playVideo(false)" :width="ytWidth"></youtube>
-            </div>
-            <div class="inputlist">
-                <span>Search for</span>
-                <autocomplete :defaultValue="this.$route.params.keyword" :search="suggest" @submit="handleSubmit"
-                              :placeholder="placeholderText"
-                              ref="searchInput"></autocomplete>
-                <span v-if="!isOneShot">up to episode </span>
-                <input v-if="!isOneShot" title="search until episode number"
-                       class="form-control" type="number" v-model="episode"
-                       min="1" :max="seriesLength">
-                <span>in</span>
-                <!--                <select title="campaign selection" class="custom-select" v-model="series">-->
-                <!--                    <option v-for="series in serverData.series" v-bind:value="series.slug">-->
-                <!--                        {{ series.title }}-->
-                <!--                    </option>-->
-                <!--                </select>-->
-                <button class="btn seriesSelectorButton" @click="showSeriesSelector=true">
-                    {{ seriesTitle }}
-                </button>
-                <button class="btn submit" @click="handleSubmit(undefined)">
-                    !
-                </button>
+    <div class="home">
+        <div class="title">
+            <h1>Critical Role Search</h1>
+            <span>Find your favourite Critical Role quote!</span>
+        </div>
+        <transition name="fade">
+            <div id="page-mask" v-if="showYtOptIn|| showSeriesSelector"></div>
+        </transition>

-            </div>
-            <b-alert v-if="error" show :variant="error.status">{{ error.message }}</b-alert>
-            <div class="entry" v-for="result in searchResult">
-                <div class="title">
-                    <div>{{ formatTimestamp(firstLine(result).starttime) }} {{ episodeName(firstLine(result)) }}</div>
-                    <div class="buttons">
-                        <router-link :to="transcriptLink(result)" class="btn" target="_blank" v-b-tooltip
-                                     title="Open this line in Transcript">
-                            <svg xmlns="http://www.w3.org/2000/svg" width="16" height="16" fill="currentColor"
-                                 class="bi bi-blockquote-left" viewBox="0 0 16 16">
-                                <path d="M2.5 3a.5.5 0 0 0 0 1h11a.5.5 0 0 0 0-1h-11zm5 3a.5.5 0 0 0 0 1h6a.5.5 0 0 0 0-1h-6zm0 3a.5.5 0 0 0 0 1h6a.5.5 0 0 0 0-1h-6zm-5 3a.5.5 0 0 0 0 1h11a.5.5 0 0 0 0-1h-11zm.79-5.373c.112-.078.26-.17.444-.275L3.524 6c-.122.074-.272.17-.452.287-.18.117-.35.26-.51.428a2.425 2.425 0 0 0-.398.562c-.11.207-.164.438-.164.692 0 .36.072.65.217.873.144.219.385.328.72.328.215 0 .383-.07.504-.211a.697.697 0 0 0 .188-.463c0-.23-.07-.404-.211-.521-.137-.121-.326-.182-.568-.182h-.282c.024-.203.065-.37.123-.498a1.38 1.38 0 0 1 .252-.37 1.94 1.94 0 0 1 .346-.298zm2.167 0c.113-.078.262-.17.445-.275L5.692 6c-.122.074-.272.17-.452.287-.18.117-.35.26-.51.428a2.425 2.425 0 0 0-.398.562c-.11.207-.164.438-.164.692 0 .36.072.65.217.873.144.219.385.328.72.328.215 0 .383-.07.504-.211a.697.697 0 0 0 .188-.463c0-.23-.07-.404-.211-.521-.137-.121-.326-.182-.568-.182h-.282a1.75 1.75 0 0 1 .118-.492c.058-.13.144-.254.257-.375a1.94 1.94 0 0 1 .346-.3z"/>
-                            </svg>
-                        </router-link>
-                        <button class="btn" @click="playVideo(result)" title="Watch line on YouTube" v-b-tooltip>
-                            <b-icon-play-fill></b-icon-play-fill>
-                        </button>
-                        <button class="btn" v-if="result.offset<10" @click="expand(result)" title="Load more context"
-                                v-b-tooltip>
-                            +
-                        </button>
-                    </div>
-                </div>
-                <p :class="{line:true,note:line.isnote,meta:line.ismeta}" :style="{borderLeftColor:getColor(line)}"
-                   v-for="line in result.lines" :key="line.id">
-                    <span v-if="line.person" class="person">{{ line.person.name }}: </span><span
-                        v-html="line.text"></span>
+        <SeriesSelector v-if="showSeriesSelector" :serverData="serverData"></SeriesSelector>
+        <div v-if="showYtOptIn" class="ytoptin popup">
+            <div>
+                <p>This play button allows you to watch the exact timestamp of this quote on YouTube.
+                    This means that the YouTube video is loaded on this page and data is sent to YouTube/Google.
+                    (<a href="https://lw1.at/i">Privacy Policy</a>)
+                </p>
+                <p>
+                    If you expected this to happen, simply continue and you won't be asked again. Otherwise you can
+                    abort or watch this timestamp directly on YouTube.
                </p>
            </div>
-            <!--            <details>-->
-            <!--                <summary>Raw Data</summary>-->
-            <!--                <pre>{{ searchResult }}</pre>-->
-            <!--            </details>-->
+            <div class="buttonrow">
+                <button class="btn" @click="showYtOptIn=false">abort</button>
+                <a class="btn" :href="ytLink" target="youtube" rel="noopener" @click="showYtOptIn=false">YouTube</a>
+                <button class="btn" @click="doYtOptIn">continue</button>
+            </div>
+        </div>
+        <div v-if="showYT" class="ytwrapper">
+            <button class="btn" @click="closeVideo">Hide</button>
+            <youtube :nocookie="true" ref="youtube" @ready="playVideo(false)" :width="ytWidth"></youtube>
+        </div>
+        <div class="inputlist">
+            <span>Search for</span>
+            <autocomplete :defaultValue="this.$route.params.keyword" :search="suggest" @submit="handleSubmit"
+                          :placeholder="placeholderText"
+                          ref="searchInput"></autocomplete>
+            <span v-if="!isOneShot">up to episode </span>
+            <input v-if="!isOneShot" title="search until episode number"
+                   class="form-control" type="number" v-model="episode"
+                   min="1" :max="seriesLength">
+            <span>in</span>
+            <!--                <select title="campaign selection" class="custom-select" v-model="series">-->
+            <!--                    <option v-for="series in serverData.series" v-bind:value="series.slug">-->
+            <!--                        {{ series.title }}-->
+            <!--                    </option>-->
+            <!--                </select>-->
+            <button class="btn seriesSelectorButton" @click="showSeriesSelector=true">
+                {{ seriesTitle }}
+            </button>
+            <button class="btn submit" @click="handleSubmit(undefined)">
+                !
+            </button>

        </div>
-        <footer>
-            <button @click="showIntro=true" class="btn btn-link">About this website</button>
-            <router-link :to="{name:'episodes'}">Episode overview</router-link>
-            <a href="https://lw1.at">My other Projects</a>
-            <a href="https://lw1.at/i">Privacy Policy</a>
-        </footer>
+        <b-alert v-if="error" show :variant="error.status">{{ error.message }}</b-alert>
+        <div class="entry" v-for="result in searchResult">
+            <div class="title">
+                <div>{{ formatTimestamp(firstLine(result).starttime) }} {{ episodeName(firstLine(result)) }}</div>
+                <div class="buttons">
+                    <router-link :to="transcriptLink(result)" class="btn" target="_blank" v-b-tooltip
+                                 title="Open this line in Transcript">
+                        <svg xmlns="http://www.w3.org/2000/svg" width="16" height="16" fill="currentColor"
+                             class="bi bi-blockquote-left" viewBox="0 0 16 16">
+                            <path d="M2.5 3a.5.5 0 0 0 0 1h11a.5.5 0 0 0 0-1h-11zm5 3a.5.5 0 0 0 0 1h6a.5.5 0 0 0 0-1h-6zm0 3a.5.5 0 0 0 0 1h6a.5.5 0 0 0 0-1h-6zm-5 3a.5.5 0 0 0 0 1h11a.5.5 0 0 0 0-1h-11zm.79-5.373c.112-.078.26-.17.444-.275L3.524 6c-.122.074-.272.17-.452.287-.18.117-.35.26-.51.428a2.425 2.425 0 0 0-.398.562c-.11.207-.164.438-.164.692 0 .36.072.65.217.873.144.219.385.328.72.328.215 0 .383-.07.504-.211a.697.697 0 0 0 .188-.463c0-.23-.07-.404-.211-.521-.137-.121-.326-.182-.568-.182h-.282c.024-.203.065-.37.123-.498a1.38 1.38 0 0 1 .252-.37 1.94 1.94 0 0 1 .346-.298zm2.167 0c.113-.078.262-.17.445-.275L5.692 6c-.122.074-.272.17-.452.287-.18.117-.35.26-.51.428a2.425 2.425 0 0 0-.398.562c-.11.207-.164.438-.164.692 0 .36.072.65.217.873.144.219.385.328.72.328.215 0 .383-.07.504-.211a.697.697 0 0 0 .188-.463c0-.23-.07-.404-.211-.521-.137-.121-.326-.182-.568-.182h-.282a1.75 1.75 0 0 1 .118-.492c.058-.13.144-.254.257-.375a1.94 1.94 0 0 1 .346-.3z"/>
+                        </svg>
+                    </router-link>
+                    <button class="btn" @click="playVideo(result)" title="Watch line on YouTube" v-b-tooltip>
+                        <b-icon-play-fill></b-icon-play-fill>
+                    </button>
+                    <button class="btn" v-if="result.offset<10" @click="expand(result)" title="Load more context"
+                            v-b-tooltip>
+                        +
+                    </button>
+                </div>
+            </div>
+            <p :class="{line:true,note:line.isnote,meta:line.ismeta}" :style="{borderLeftColor:getColor(line)}"
+               v-for="line in result.lines" :key="line.id">
+                <span v-if="line.person" class="person">{{ line.person.name }}: </span><span
+                    v-html="line.text"></span>
+            </p>
+        </div>
+        <!--            <details>-->
+        <!--                <summary>Raw Data</summary>-->
+        <!--                <pre>{{ searchResult }}</pre>-->
+        <!--            </details>-->
+
    </div>
 </template>

@ -136,7 +128,6 @@ export default Vue.extend({
      showYT: false,
      ytResult: undefined as Result | undefined,
      ytWidth: 640,
-      showIntro: true,
      showSeriesSelector: false,
      placeholderText: "",
      placeholderFullText: "",
@ -156,9 +147,6 @@ export default Vue.extend({
    clearTimeout(this.placeholderTimeout);
  },
  mounted(): void {
-    if (localStorage.getItem("showIntro") !== null) {
-      this.showIntro = localStorage.getItem("showIntro") === "true";
-    }
    if (localStorage.getItem("ytOptIn") !== null) {
      this.ytOptIn = localStorage.getItem("ytOptIn") === "true";
    }
@ -422,9 +410,6 @@ export default Vue.extend({
    ytOptIn(value: boolean): void {
      localStorage.setItem("ytOption", value.toString());
    },
-    showIntro(value: boolean): void {
-      localStorage.setItem("showIntro", value.toString());
-    }
  },
 });
 </script>