allow updated subtitles to invalidate existing data

fetch.py

@@ -1,11 +1,12 @@
+import hashlib
 import re
 from subprocess import run
 
 import youtube_dl
 from peewee import DoesNotExist
 
-from models import Episode, Series
-from utils import srtdir
+from models import Episode, Series, Line, Phrase
+from utils import srtdir, pretty_title
 
 series_data = [
     {
@@ -66,6 +67,7 @@ def main():
                 e.series = s
                 e.video_number = nr
             e.title = video["title"]
+            e.pretty_title = pretty_title(video["title"])
             if s.is_campaign:
                 try:
                     match = regex.search(video["title"])
@@ -81,17 +83,28 @@ def main():
             e.save()
             vttfile = srtdir / str(e.id)
             ydl_opts["outtmpl"] = str(vttfile)
-            if e.downloaded:
-                continue
             with youtube_dl.YoutubeDL(ydl_opts) as ydl:
                 ydl.download([f'https://www.youtube.com/watch?v={e.youtube_id}'])
-                run(["ffmpeg", "-y", "-i", vttfile.with_suffix(".en.vtt"), vttfile.with_suffix(".srt")])
-                e.downloaded = True
-                try:
-                    vttfile.with_suffix(".en.vtt").unlink()
-                except FileNotFoundError:
-                    e.downloaded = False
-                e.save()
+            run(["ffmpeg", "-y", "-i", vttfile.with_suffix(".en.vtt"), vttfile.with_suffix(".srt")])
+            e.downloaded = True
+            try:
+                vttfile.with_suffix(".en.vtt").unlink()
+                with vttfile.with_suffix(".srt").open("rb") as f:
+                    file_hash = hashlib.sha256()
+                    while True:
+                        chunk = f.read(8192)
+                        if not chunk:
+                            break
+                        file_hash.update(chunk)
+                if e.subtitle_hash != file_hash.hexdigest():
+                    Line.delete().where(Line.episode == e)
+                    Phrase.delete().where(Phrase.episode == e)
+                    e.phrases_imported = False
+                    e.text_imported = False
+                    e.subtitle_hash = file_hash.hexdigest()
+            except FileNotFoundError:
+                e.downloaded = False
+            e.save()
 
 
 if __name__ == '__main__':

models.py

@@ -1,4 +1,6 @@
-from peewee import PostgresqlDatabase, Model, IntegerField, CharField, BooleanField, ForeignKeyField
+from datetime import datetime
+
+from peewee import PostgresqlDatabase, Model, IntegerField, CharField, BooleanField, ForeignKeyField, DateTimeField
 from playhouse.postgres_ext import TSVectorField
 
 from config import dbauth
@@ -24,9 +26,12 @@ class Episode(BaseModel):
     video_number = IntegerField()
     youtube_id = CharField(max_length=11)
     title = CharField(max_length=100)
+    pretty_title = CharField(max_length=100)
     downloaded = BooleanField(default=False)
     text_imported = BooleanField(default=False)
     phrases_imported = BooleanField(default=False)
+    subtitle_hash = CharField(max_length=64)
+    last_updated = DateTimeField(default=datetime.now)
 
     class Meta:
         indexes = ((("series", "video_number"), True),)

server.py

@@ -21,7 +21,7 @@ def add_cors(response):
     return response
 
 
-global_excludes = [Line.search_text, Episode.phrases_imported, Episode.text_imported, Person.series]
+global_excludes = [Line.search_text, Episode.phrases_imported, Episode.text_imported, Person.series, Episode.title]
 
 
 @app.route("/api/suggest")
@@ -116,7 +116,7 @@ def expand():
 
 @app.route("/api/episodes")
 def episodes():
-    all_series: List[Series] = Series.select()
+    all_series: List[Series] = Series.select().order_by(Series.id)
     data = []
     for series in all_series:
 
@@ -124,8 +124,7 @@ def episodes():
 
         series_data = []
         for episode in episodes:
-            entry = model_to_dict(episode, exclude=[Episode.series])
-            entry["title"] = entry["title"].split("|")[0].strip()
+            entry = model_to_dict(episode, exclude=[Episode.series, Episode.title])
             series_data.append(entry)
         data.append({
             "meta": model_to_dict(series),

utils.py

@@ -20,3 +20,14 @@ def episode_speaker(series_title: str, episode: int) -> Optional[str]:
     if episode in series:
         return series[episode]
     return None
+
+
+def pretty_title(title: str) -> str:
+    if "|" in title:
+        return title.split("|")[0].strip()
+    if "Handbooker Helper:" in title:
+        return title.replace("Handbooker Helper:", "").strip()
+    if "Critical Role RPG Show" in title:
+        return title.split("-")[0].strip()
+    else:
+        return title.strip()

web/package.json

@@ -4,16 +4,16 @@
   "private": true,
   "scripts": {
     "serve": "vue-cli-service serve",
-    "build": "vue-cli-service build",
+    "build": "vue-cli-service build --modern",
     "lint": "vue-cli-service lint"
   },
   "dependencies": {
     "@trevoreyre/autocomplete-vue": "^2.1.1",
-    "@types/lodash": "^4.14.149",
+    "@types/lodash-es": "^4.17.3",
     "bootstrap": "^4.4.1",
     "bootstrap-vue": "^2.5.0",
     "core-js": "^3.6.4",
-    "lodash": "^4.17.15",
+    "lodash-es": "^4.17.15",
     "vue": "^2.6.10",
     "vue-router": "^3.0.3",
     "vue-youtube": "^1.4.0"
@@ -33,7 +33,8 @@
     }
   },
   "browserslist": [
-    "> 1%",
-    "last 2 versions"
+    "> 1% in AT",
+    "last 2 versions",
+    "not ie <= 11"
   ]
 }

web/src/interfaces.ts

@@ -15,7 +15,7 @@ export interface Episode {
   "episode_number": number;
   "id": number;
   "series": Series;
-  "title": string;
+  "pretty_title": string;
   "video_number": number;
   "youtube_id": string;
 }

web/src/main.ts

@@ -1,7 +1,6 @@
 import Vue from "vue";
 import App from "./App.vue";
 import router from "./router";
-import {BootstrapVue, IconsPlugin} from "bootstrap-vue";
 import "./custom.scss";
 
 Vue.config.productionTip = false;
@@ -11,4 +10,3 @@ new Vue({
   render: (h) => h(App),
 }).$mount("#app");
 
-Vue.use(BootstrapVue);

web/src/views/Episodes.vue

@@ -22,7 +22,7 @@
                 <tbody>
                 <tr v-for="episode in series.episodes" :key="episode.id">
                     <td><a :href="youtubeLink(episode)" target="youtube" rel="noopener">
-                        {{ episode.title }}
+                        {{ episode.pretty_title }}
                     </a>
                     </td>
                     <td>{{ episode.episode_number }}</td>
@@ -40,7 +40,6 @@
                 </tbody>
             </table>
         </div>
-        <pre>{{ series_data }}</pre>
     </div>
 </template>
 

web/src/views/Home.vue

@@ -124,7 +124,7 @@ import {Line, Result, ServerData, ServerMessage} from "@/interfaces";
 import {BAlert, BIcon, BIconPlayFill} from "bootstrap-vue";
 // @ts-ignore
 import VueYoutube from "vue-youtube";
-import {debounce} from "lodash";
+import debounce from "lodash-es/debounce";
 
 import {baseURL} from "@/utils";
 
@@ -234,9 +234,9 @@ export default Vue.extend({
     },
     episodeName(line: Line): string {
       if (line.episode.series.is_campaign) {
-        return `Episode ${line.episode.episode_number}`;
+        return `Episode ${line.episode.episode_number}: ${line.episode.pretty_title}`;
       }
-      return line.episode.title;
+      return line.episode.pretty_title;
     },
     formatTimestamp(ts: number) {
       return new Date(ts).toISOString().substr(11, 8);

web/yarn.lock

@@ -963,7 +963,14 @@
   resolved "https://registry.yarnpkg.com/@types/json-schema/-/json-schema-7.0.5.tgz#dcce4430e64b443ba8945f0290fb564ad5bac6dd"
   integrity sha512-7+2BITlgjgDhH0vvwZU/HZJVyk+2XUlvxXe8dFMedNX/aMkaOq++rMAFXc0tM7ij15QaWlbdQASBR9dihi+bDQ==
 
-"@types/lodash@^4.14.149":
+"@types/lodash-es@^4.17.3":
+  version "4.17.3"
+  resolved "https://registry.yarnpkg.com/@types/lodash-es/-/lodash-es-4.17.3.tgz#87eb0b3673b076b8ee655f1890260a136af09a2d"
+  integrity sha512-iHI0i7ZAL1qepz1Y7f3EKg/zUMDwDfTzitx+AlHhJJvXwenP682ZyGbgPSc5Ej3eEAKVbNWKFuwOadCj5vBbYQ==
+  dependencies:
+    "@types/lodash" "*"
+
+"@types/lodash@*":
   version "4.14.159"
   resolved "https://registry.yarnpkg.com/@types/lodash/-/lodash-4.14.159.tgz#61089719dc6fdd9c5cb46efc827f2571d1517065"
   integrity sha512-gF7A72f7WQN33DpqOWw9geApQPh4M3PxluMtaHxWHXEGSN12/WbcEk/eNSqWNQcQhF66VSZ06vCF94CrHwXJDg==
@@ -4930,6 +4937,11 @@ locate-path@^5.0.0:
   dependencies:
     p-locate "^4.1.0"
 
+lodash-es@^4.17.15:
+  version "4.17.15"
+  resolved "https://registry.yarnpkg.com/lodash-es/-/lodash-es-4.17.15.tgz#21bd96839354412f23d7a10340e5eac6ee455d78"
+  integrity sha512-rlrc3yU3+JNOpZ9zj5pQtxnx2THmvRykwL4Xlxoa8I9lHBlVbbyPhgyPMioxVZ4NqyxaVVtaJnzsyOidQIhyyQ==
+
 lodash.defaultsdeep@^4.6.1:
   version "4.6.1"
   resolved "https://registry.yarnpkg.com/lodash.defaultsdeep/-/lodash.defaultsdeep-4.6.1.tgz#512e9bd721d272d94e3d3a63653fa17516741ca6"