unfinished archive.org url check

commands/archive_sources.ts

@@ -0,0 +1,68 @@
+import {Crossing} from "../interfaces";
+import fs from "fs";
+import axios, {AxiosResponse} from "axios";
+
+interface WBMSnapshot {
+    available: boolean
+    url: string
+    timestamp: string
+    status: string
+}
+
+interface WBMResponse {
+    url: string
+    archived_snapshots: { closest: WBMSnapshot }
+}
+
+const files = ["Wien", "Burgenland", "Kärnten", "Niederösterreich", "Oberösterreich", "Salzburg", "Steiermark", "Tirol", "Vorarlberg"]
+
+let urls: string[] = []
+
+function checkURLs() {
+    files.forEach(f => {
+        const data: Crossing[] = JSON.parse(fs.readFileSync(`../data/${f}.json`, 'utf8'));
+        data.forEach(cr => {
+            cr.sources.forEach(s => {
+                if (typeof s.url === "undefined") {
+                    return
+                }
+                urls.push(s.url)
+            })
+        })
+    })
+
+
+    // urls = [urls[0]]
+    let requests: Promise<AxiosResponse<any, any>>[] = []
+
+    urls.forEach(url => {
+        console.log("https://web.archive.org/save/"+url)
+        // requests.push(axios.get("https://archive.org/wayback/available", {params: {url}}))
+    })
+    process.exit()
+
+    axios.all(requests).then(responses => {
+        responses.forEach(r => {
+            let archived: boolean = false
+            const data: WBMResponse = r.data
+            console.info(data)
+            if (Object.keys(data.archived_snapshots).length === 0) {
+                archived = false;
+                console.log(data.url)
+                return
+            }
+            const snapshot = data.archived_snapshots.closest
+            if (snapshot.status != "200" || !snapshot.available) {
+                archived = false;
+                console.log(data.url)
+                return
+
+            }
+            // console.warn(data.url)
+
+
+        })
+    })
+}
+
+checkURLs()

data/Wien.json

@@ -1128,7 +1128,7 @@
       },
       {
         "type": "official",
-        "url": "https://www.doebling.spoe.wien/2022/05/26/regenbogenzebrastreifen/",
+        "url": "https://doebling.spoe.wien/artikel/regenbogenzebrastreifen",
         "date": "2022-05-26"
       }
     ],

package.json

@@ -5,6 +5,7 @@
   "scripts": {
     "fetch": "cd commands && ts-node fetch.ts",
     "stats": "cd commands && ts-node stats.ts",
+    "archive": "cd commands && ts-node archive_sources.ts",
     "dev": "vite",
     "build": "tsc --skipLibCheck && vite build",
     "preview": "vite preview",

tsconfig.json

@@ -91,7 +91,7 @@
     // "exactOptionalPropertyTypes": true,               /* Interpret optional property types as written, rather than adding 'undefined'. */
     // "noImplicitReturns": true,                        /* Enable error reporting for codepaths that do not explicitly return in a function. */
     // "noFallthroughCasesInSwitch": true,               /* Enable error reporting for fallthrough cases in switch statements. */
-    // "noUncheckedIndexedAccess": true,                 /* Add 'undefined' to a type when accessed using an index. */
+     "noUncheckedIndexedAccess": true,                 /* Add 'undefined' to a type when accessed using an index. */
     // "noImplicitOverride": true,                       /* Ensure overriding members in derived classes are marked with an override modifier. */
     // "noPropertyAccessFromIndexSignature": true,       /* Enforces using indexed accessors for keys declared using an indexed type. */
     // "allowUnusedLabels": true,                        /* Disable error reporting for unused labels. */