mirror of
https://github.com/Findus23/RainbowRoad.git
synced 2024-09-19 16:03:52 +02:00
unfinished archive.org url check
This commit is contained in:
parent
5243a708d6
commit
20e2dfa5ce
4 changed files with 71 additions and 2 deletions
68
commands/archive_sources.ts
Normal file
68
commands/archive_sources.ts
Normal file
|
@ -0,0 +1,68 @@
|
||||||
|
import {Crossing} from "../interfaces";
|
||||||
|
import fs from "fs";
|
||||||
|
import axios, {AxiosResponse} from "axios";
|
||||||
|
|
||||||
|
interface WBMSnapshot {
|
||||||
|
available: boolean
|
||||||
|
url: string
|
||||||
|
timestamp: string
|
||||||
|
status: string
|
||||||
|
}
|
||||||
|
|
||||||
|
interface WBMResponse {
|
||||||
|
url: string
|
||||||
|
archived_snapshots: { closest: WBMSnapshot }
|
||||||
|
}
|
||||||
|
|
||||||
|
const files = ["Wien", "Burgenland", "Kärnten", "Niederösterreich", "Oberösterreich", "Salzburg", "Steiermark", "Tirol", "Vorarlberg"]
|
||||||
|
|
||||||
|
let urls: string[] = []
|
||||||
|
|
||||||
|
function checkURLs() {
|
||||||
|
files.forEach(f => {
|
||||||
|
const data: Crossing[] = JSON.parse(fs.readFileSync(`../data/${f}.json`, 'utf8'));
|
||||||
|
data.forEach(cr => {
|
||||||
|
cr.sources.forEach(s => {
|
||||||
|
if (typeof s.url === "undefined") {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
urls.push(s.url)
|
||||||
|
})
|
||||||
|
})
|
||||||
|
})
|
||||||
|
|
||||||
|
|
||||||
|
// urls = [urls[0]]
|
||||||
|
let requests: Promise<AxiosResponse<any, any>>[] = []
|
||||||
|
|
||||||
|
urls.forEach(url => {
|
||||||
|
console.log("https://web.archive.org/save/"+url)
|
||||||
|
// requests.push(axios.get("https://archive.org/wayback/available", {params: {url}}))
|
||||||
|
})
|
||||||
|
process.exit()
|
||||||
|
|
||||||
|
axios.all(requests).then(responses => {
|
||||||
|
responses.forEach(r => {
|
||||||
|
let archived: boolean = false
|
||||||
|
const data: WBMResponse = r.data
|
||||||
|
console.info(data)
|
||||||
|
if (Object.keys(data.archived_snapshots).length === 0) {
|
||||||
|
archived = false;
|
||||||
|
console.log(data.url)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
const snapshot = data.archived_snapshots.closest
|
||||||
|
if (snapshot.status != "200" || !snapshot.available) {
|
||||||
|
archived = false;
|
||||||
|
console.log(data.url)
|
||||||
|
return
|
||||||
|
|
||||||
|
}
|
||||||
|
// console.warn(data.url)
|
||||||
|
|
||||||
|
|
||||||
|
})
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
checkURLs()
|
|
@ -1128,7 +1128,7 @@
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
"type": "official",
|
"type": "official",
|
||||||
"url": "https://www.doebling.spoe.wien/2022/05/26/regenbogenzebrastreifen/",
|
"url": "https://doebling.spoe.wien/artikel/regenbogenzebrastreifen",
|
||||||
"date": "2022-05-26"
|
"date": "2022-05-26"
|
||||||
}
|
}
|
||||||
],
|
],
|
||||||
|
|
|
@ -5,6 +5,7 @@
|
||||||
"scripts": {
|
"scripts": {
|
||||||
"fetch": "cd commands && ts-node fetch.ts",
|
"fetch": "cd commands && ts-node fetch.ts",
|
||||||
"stats": "cd commands && ts-node stats.ts",
|
"stats": "cd commands && ts-node stats.ts",
|
||||||
|
"archive": "cd commands && ts-node archive_sources.ts",
|
||||||
"dev": "vite",
|
"dev": "vite",
|
||||||
"build": "tsc --skipLibCheck && vite build",
|
"build": "tsc --skipLibCheck && vite build",
|
||||||
"preview": "vite preview",
|
"preview": "vite preview",
|
||||||
|
|
|
@ -91,7 +91,7 @@
|
||||||
// "exactOptionalPropertyTypes": true, /* Interpret optional property types as written, rather than adding 'undefined'. */
|
// "exactOptionalPropertyTypes": true, /* Interpret optional property types as written, rather than adding 'undefined'. */
|
||||||
// "noImplicitReturns": true, /* Enable error reporting for codepaths that do not explicitly return in a function. */
|
// "noImplicitReturns": true, /* Enable error reporting for codepaths that do not explicitly return in a function. */
|
||||||
// "noFallthroughCasesInSwitch": true, /* Enable error reporting for fallthrough cases in switch statements. */
|
// "noFallthroughCasesInSwitch": true, /* Enable error reporting for fallthrough cases in switch statements. */
|
||||||
// "noUncheckedIndexedAccess": true, /* Add 'undefined' to a type when accessed using an index. */
|
"noUncheckedIndexedAccess": true, /* Add 'undefined' to a type when accessed using an index. */
|
||||||
// "noImplicitOverride": true, /* Ensure overriding members in derived classes are marked with an override modifier. */
|
// "noImplicitOverride": true, /* Ensure overriding members in derived classes are marked with an override modifier. */
|
||||||
// "noPropertyAccessFromIndexSignature": true, /* Enforces using indexed accessors for keys declared using an indexed type. */
|
// "noPropertyAccessFromIndexSignature": true, /* Enforces using indexed accessors for keys declared using an indexed type. */
|
||||||
// "allowUnusedLabels": true, /* Disable error reporting for unused labels. */
|
// "allowUnusedLabels": true, /* Disable error reporting for unused labels. */
|
||||||
|
|
Loading…
Reference in a new issue