mirror of
https://github.com/Findus23/cr-search.git
synced 2024-09-19 15:23:44 +02:00
allow updated subtitles to invalidate existing data
This commit is contained in:
parent
f2dbfd0bf0
commit
d996996531
10 changed files with 68 additions and 30 deletions
21
fetch.py
21
fetch.py
|
@ -1,11 +1,12 @@
|
|||
import hashlib
|
||||
import re
|
||||
from subprocess import run
|
||||
|
||||
import youtube_dl
|
||||
from peewee import DoesNotExist
|
||||
|
||||
from models import Episode, Series
|
||||
from utils import srtdir
|
||||
from models import Episode, Series, Line, Phrase
|
||||
from utils import srtdir, pretty_title
|
||||
|
||||
series_data = [
|
||||
{
|
||||
|
@ -66,6 +67,7 @@ def main():
|
|||
e.series = s
|
||||
e.video_number = nr
|
||||
e.title = video["title"]
|
||||
e.pretty_title = pretty_title(video["title"])
|
||||
if s.is_campaign:
|
||||
try:
|
||||
match = regex.search(video["title"])
|
||||
|
@ -81,14 +83,25 @@ def main():
|
|||
e.save()
|
||||
vttfile = srtdir / str(e.id)
|
||||
ydl_opts["outtmpl"] = str(vttfile)
|
||||
if e.downloaded:
|
||||
continue
|
||||
with youtube_dl.YoutubeDL(ydl_opts) as ydl:
|
||||
ydl.download([f'https://www.youtube.com/watch?v={e.youtube_id}'])
|
||||
run(["ffmpeg", "-y", "-i", vttfile.with_suffix(".en.vtt"), vttfile.with_suffix(".srt")])
|
||||
e.downloaded = True
|
||||
try:
|
||||
vttfile.with_suffix(".en.vtt").unlink()
|
||||
with vttfile.with_suffix(".srt").open("rb") as f:
|
||||
file_hash = hashlib.sha256()
|
||||
while True:
|
||||
chunk = f.read(8192)
|
||||
if not chunk:
|
||||
break
|
||||
file_hash.update(chunk)
|
||||
if e.subtitle_hash != file_hash.hexdigest():
|
||||
Line.delete().where(Line.episode == e)
|
||||
Phrase.delete().where(Phrase.episode == e)
|
||||
e.phrases_imported = False
|
||||
e.text_imported = False
|
||||
e.subtitle_hash = file_hash.hexdigest()
|
||||
except FileNotFoundError:
|
||||
e.downloaded = False
|
||||
e.save()
|
||||
|
|
|
@ -1,4 +1,6 @@
|
|||
from peewee import PostgresqlDatabase, Model, IntegerField, CharField, BooleanField, ForeignKeyField
|
||||
from datetime import datetime
|
||||
|
||||
from peewee import PostgresqlDatabase, Model, IntegerField, CharField, BooleanField, ForeignKeyField, DateTimeField
|
||||
from playhouse.postgres_ext import TSVectorField
|
||||
|
||||
from config import dbauth
|
||||
|
@ -24,9 +26,12 @@ class Episode(BaseModel):
|
|||
video_number = IntegerField()
|
||||
youtube_id = CharField(max_length=11)
|
||||
title = CharField(max_length=100)
|
||||
pretty_title = CharField(max_length=100)
|
||||
downloaded = BooleanField(default=False)
|
||||
text_imported = BooleanField(default=False)
|
||||
phrases_imported = BooleanField(default=False)
|
||||
subtitle_hash = CharField(max_length=64)
|
||||
last_updated = DateTimeField(default=datetime.now)
|
||||
|
||||
class Meta:
|
||||
indexes = ((("series", "video_number"), True),)
|
||||
|
|
|
@ -21,7 +21,7 @@ def add_cors(response):
|
|||
return response
|
||||
|
||||
|
||||
global_excludes = [Line.search_text, Episode.phrases_imported, Episode.text_imported, Person.series]
|
||||
global_excludes = [Line.search_text, Episode.phrases_imported, Episode.text_imported, Person.series, Episode.title]
|
||||
|
||||
|
||||
@app.route("/api/suggest")
|
||||
|
@ -116,7 +116,7 @@ def expand():
|
|||
|
||||
@app.route("/api/episodes")
|
||||
def episodes():
|
||||
all_series: List[Series] = Series.select()
|
||||
all_series: List[Series] = Series.select().order_by(Series.id)
|
||||
data = []
|
||||
for series in all_series:
|
||||
|
||||
|
@ -124,8 +124,7 @@ def episodes():
|
|||
|
||||
series_data = []
|
||||
for episode in episodes:
|
||||
entry = model_to_dict(episode, exclude=[Episode.series])
|
||||
entry["title"] = entry["title"].split("|")[0].strip()
|
||||
entry = model_to_dict(episode, exclude=[Episode.series, Episode.title])
|
||||
series_data.append(entry)
|
||||
data.append({
|
||||
"meta": model_to_dict(series),
|
||||
|
|
11
utils.py
11
utils.py
|
@ -20,3 +20,14 @@ def episode_speaker(series_title: str, episode: int) -> Optional[str]:
|
|||
if episode in series:
|
||||
return series[episode]
|
||||
return None
|
||||
|
||||
|
||||
def pretty_title(title: str) -> str:
|
||||
if "|" in title:
|
||||
return title.split("|")[0].strip()
|
||||
if "Handbooker Helper:" in title:
|
||||
return title.replace("Handbooker Helper:", "").strip()
|
||||
if "Critical Role RPG Show" in title:
|
||||
return title.split("-")[0].strip()
|
||||
else:
|
||||
return title.strip()
|
||||
|
|
|
@ -4,16 +4,16 @@
|
|||
"private": true,
|
||||
"scripts": {
|
||||
"serve": "vue-cli-service serve",
|
||||
"build": "vue-cli-service build",
|
||||
"build": "vue-cli-service build --modern",
|
||||
"lint": "vue-cli-service lint"
|
||||
},
|
||||
"dependencies": {
|
||||
"@trevoreyre/autocomplete-vue": "^2.1.1",
|
||||
"@types/lodash": "^4.14.149",
|
||||
"@types/lodash-es": "^4.17.3",
|
||||
"bootstrap": "^4.4.1",
|
||||
"bootstrap-vue": "^2.5.0",
|
||||
"core-js": "^3.6.4",
|
||||
"lodash": "^4.17.15",
|
||||
"lodash-es": "^4.17.15",
|
||||
"vue": "^2.6.10",
|
||||
"vue-router": "^3.0.3",
|
||||
"vue-youtube": "^1.4.0"
|
||||
|
@ -33,7 +33,8 @@
|
|||
}
|
||||
},
|
||||
"browserslist": [
|
||||
"> 1%",
|
||||
"last 2 versions"
|
||||
"> 1% in AT",
|
||||
"last 2 versions",
|
||||
"not ie <= 11"
|
||||
]
|
||||
}
|
||||
|
|
|
@ -15,7 +15,7 @@ export interface Episode {
|
|||
"episode_number": number;
|
||||
"id": number;
|
||||
"series": Series;
|
||||
"title": string;
|
||||
"pretty_title": string;
|
||||
"video_number": number;
|
||||
"youtube_id": string;
|
||||
}
|
||||
|
|
|
@ -1,7 +1,6 @@
|
|||
import Vue from "vue";
|
||||
import App from "./App.vue";
|
||||
import router from "./router";
|
||||
import {BootstrapVue, IconsPlugin} from "bootstrap-vue";
|
||||
import "./custom.scss";
|
||||
|
||||
Vue.config.productionTip = false;
|
||||
|
@ -11,4 +10,3 @@ new Vue({
|
|||
render: (h) => h(App),
|
||||
}).$mount("#app");
|
||||
|
||||
Vue.use(BootstrapVue);
|
||||
|
|
|
@ -22,7 +22,7 @@
|
|||
<tbody>
|
||||
<tr v-for="episode in series.episodes" :key="episode.id">
|
||||
<td><a :href="youtubeLink(episode)" target="youtube" rel="noopener">
|
||||
{{ episode.title }}
|
||||
{{ episode.pretty_title }}
|
||||
</a>
|
||||
</td>
|
||||
<td>{{ episode.episode_number }}</td>
|
||||
|
@ -40,7 +40,6 @@
|
|||
</tbody>
|
||||
</table>
|
||||
</div>
|
||||
<pre>{{ series_data }}</pre>
|
||||
</div>
|
||||
</template>
|
||||
|
||||
|
|
|
@ -124,7 +124,7 @@ import {Line, Result, ServerData, ServerMessage} from "@/interfaces";
|
|||
import {BAlert, BIcon, BIconPlayFill} from "bootstrap-vue";
|
||||
// @ts-ignore
|
||||
import VueYoutube from "vue-youtube";
|
||||
import {debounce} from "lodash";
|
||||
import debounce from "lodash-es/debounce";
|
||||
|
||||
import {baseURL} from "@/utils";
|
||||
|
||||
|
@ -234,9 +234,9 @@ export default Vue.extend({
|
|||
},
|
||||
episodeName(line: Line): string {
|
||||
if (line.episode.series.is_campaign) {
|
||||
return `Episode ${line.episode.episode_number}`;
|
||||
return `Episode ${line.episode.episode_number}: ${line.episode.pretty_title}`;
|
||||
}
|
||||
return line.episode.title;
|
||||
return line.episode.pretty_title;
|
||||
},
|
||||
formatTimestamp(ts: number) {
|
||||
return new Date(ts).toISOString().substr(11, 8);
|
||||
|
|
|
@ -963,7 +963,14 @@
|
|||
resolved "https://registry.yarnpkg.com/@types/json-schema/-/json-schema-7.0.5.tgz#dcce4430e64b443ba8945f0290fb564ad5bac6dd"
|
||||
integrity sha512-7+2BITlgjgDhH0vvwZU/HZJVyk+2XUlvxXe8dFMedNX/aMkaOq++rMAFXc0tM7ij15QaWlbdQASBR9dihi+bDQ==
|
||||
|
||||
"@types/lodash@^4.14.149":
|
||||
"@types/lodash-es@^4.17.3":
|
||||
version "4.17.3"
|
||||
resolved "https://registry.yarnpkg.com/@types/lodash-es/-/lodash-es-4.17.3.tgz#87eb0b3673b076b8ee655f1890260a136af09a2d"
|
||||
integrity sha512-iHI0i7ZAL1qepz1Y7f3EKg/zUMDwDfTzitx+AlHhJJvXwenP682ZyGbgPSc5Ej3eEAKVbNWKFuwOadCj5vBbYQ==
|
||||
dependencies:
|
||||
"@types/lodash" "*"
|
||||
|
||||
"@types/lodash@*":
|
||||
version "4.14.159"
|
||||
resolved "https://registry.yarnpkg.com/@types/lodash/-/lodash-4.14.159.tgz#61089719dc6fdd9c5cb46efc827f2571d1517065"
|
||||
integrity sha512-gF7A72f7WQN33DpqOWw9geApQPh4M3PxluMtaHxWHXEGSN12/WbcEk/eNSqWNQcQhF66VSZ06vCF94CrHwXJDg==
|
||||
|
@ -4930,6 +4937,11 @@ locate-path@^5.0.0:
|
|||
dependencies:
|
||||
p-locate "^4.1.0"
|
||||
|
||||
lodash-es@^4.17.15:
|
||||
version "4.17.15"
|
||||
resolved "https://registry.yarnpkg.com/lodash-es/-/lodash-es-4.17.15.tgz#21bd96839354412f23d7a10340e5eac6ee455d78"
|
||||
integrity sha512-rlrc3yU3+JNOpZ9zj5pQtxnx2THmvRykwL4Xlxoa8I9lHBlVbbyPhgyPMioxVZ4NqyxaVVtaJnzsyOidQIhyyQ==
|
||||
|
||||
lodash.defaultsdeep@^4.6.1:
|
||||
version "4.6.1"
|
||||
resolved "https://registry.yarnpkg.com/lodash.defaultsdeep/-/lodash.defaultsdeep-4.6.1.tgz#512e9bd721d272d94e3d3a63653fa17516741ca6"
|
||||
|
|
Loading…
Reference in a new issue