improved favicon search

analyseIco.py

@@ -10,10 +10,11 @@ icons = output.splitlines()
 regex = b"\d+x\d+"
 maxsize = 0
 maxpos = 0
-for i, icon in enumerate(icons):
-    resolution = re.findall(regex, icon)[0].decode()  # e.g. 16x16
+for i, file_line in enumerate(icons):
+    resolution = re.findall(regex, file_line)[0].decode()  # e.g. 16x16
     size = int(resolution.split("x")[0])  # e.g. 16
-    if size >= maxsize:
+    # "1-bit"-images are not very helpful
+    if size >= maxsize and " 1-bit" not in str(file_line):
         maxsize = size
         maxpos = i
 print(maxpos)

convert.sh

@@ -16,7 +16,7 @@ for i in src/**/*.{png,gif,jpg,ico}; do
     fi
     if [[ $i == *.ico ]]
     then
-        if file "$i" | grep -E "HTML|empty|  data" # if no valid image
+        if file "$i" | grep -E "HTML|empty|ASCII text|: data|SVG" # if no valid image
         then
             rm "$i"
         else
@@ -30,26 +30,29 @@ for i in src/**/*.{png,gif,jpg,ico}; do
             i=$newIcon
         fi
     fi
-    convert \
-        "$i" \
-        -strip \
-        -transparent white \
-        -background none \
-        -trim \
-        -resize ${size}x${size} \
-        -gravity center \
-        -extent ${size}x${size} \
-        "$distFile"
-        # input file
-        # strip metadata
-        # make background transparent
-        # keep transparency
-        # cut border
-        # get only one image from .ico
-        # resize while keeping the aspect ratio
-        # center image
-        # fit to 16x16
-    # optimize png:
-    pngquant -f --ext .png -s 1 --skip-if-larger --quality 70-95 "$distFile"
-    echo ""
+    # if file (or symlink) -> didn't get deleted
+    if [ -e "$i" ]
+    then
+        convert \
+            "$i" \
+            -strip \
+            -transparent white \
+            -background none \
+            -trim \
+            -resize ${size}x${size} \
+            -gravity center \
+            -extent ${size}x${size} \
+            "$distFile"
+            # input file
+            # strip metadata
+            # make background transparent
+            # keep transparency
+            # cut border
+            # get only one image from .ico
+            # resize while keeping the aspect ratio
+            # center image
+            # fit to 16x16
+        # optimize png:
+        pngquant -f --ext .png -s 1 --skip-if-larger --quality 70-95 "$distFile"
+    fi
 done

referrers.py

@@ -21,6 +21,10 @@ def download_favicon(homepage_html, url):
     favicon_element = soup.find("link", rel="shortcut icon")
     if favicon_element and "href" in favicon_element:
         favicon_path = favicon_element['href']
+    elif soup.find("link", rel="icon") and "href" in soup.find("link", rel="icon"):
+        # some sites don't use "shortcut icon" for favicon
+        # in this case we take the first other icon and hope it fits
+        favicon_path = soup.find("link", rel="icon")["href"]
     else:
         favicon_path = "/favicon.ico"
     print(favicon_path)
@@ -34,7 +38,7 @@ def download_favicon(homepage_html, url):
                 r.raw.decode_content = True
                 shutil.copyfileobj(r.raw, f)
 
-    except requests.exceptions.ConnectionError:
+    except (requests.exceptions.ConnectionError, requests.exceptions.InvalidSchema):
         print("Error while downloading favicon")
 
 
@@ -77,7 +81,7 @@ def main(search_engines):
 
 
 if __name__ == "__main__":
-    MODE = sys.argv[1]
+    MODE = sys.argv[1] if len(sys.argv) >= 2 else ""
 
     if MODE == "searchengines":
         yamlfile = "vendor/piwik/searchengine-and-social-list/SearchEngines.yml"