rewrite test script

.travis.yml

@@ -2,7 +2,7 @@ os: linux
 dist: bionic
 language: python
 python:
-  "3.8"
+  "3.9"
 
 before_install:
   - git clone https://github.com/piwik/piwik-package.git tmp/piwik-package

requirements.txt

@@ -1,2 +1,2 @@
-Pillow==8.1.2
-PyYAML==5.4.1
+Pillow==9.0.1
+PyYAML==6.0

tests.py

@@ -16,9 +16,10 @@ import hashlib
 import json
 import os
 import re
+import subprocess
 import sys
-from glob import glob
-from subprocess import Popen, PIPE
+from pathlib import Path
+from typing import Dict, List, Iterator, Iterable
 from urllib.parse import urlparse
 
 import yaml
@@ -30,104 +31,112 @@ min_image_size = 48
 
 placeholder_icon_hash = "398a623a3b0b10eba6d1884b0ff1713ee12aeafaa8efaf67b60a4624f4dce48c"
 
-searchEnginesFile = "vendor/matomo/searchengine-and-social-list/SearchEngines.yml"
-socialsEnginesFile = "vendor/matomo/searchengine-and-social-list/Socials.yml"
+searchEnginesFile = Path("vendor/matomo/searchengine-and-social-list/SearchEngines.yml")
+socialsEnginesFile = Path("vendor/matomo/searchengine-and-social-list/Socials.yml")
+build_script_file = Path("tmp/piwik-package/scripts/build-package.sh")
+
+src = Path("src/")
+dist: Path = Path("dist/")
 
 
-def print_warning(string):
+def print_warning(string: str) -> None:
     print("\033[33;1m⚠\033[0m " + string)
 
 
-def print_error(string):
+def print_error(string: str) -> None:
     print("\033[31;1m⚠ " + string + "\033[0m")
 
 
-def load_yaml(file):
-    with open(file, 'r') as stream:
+def load_yaml(file: Path):
+    with file.open() as stream:
         return yaml.safe_load(stream)
 
 
-def image_exists(pathslug):
+def image_exists(pathslug: Path) -> bool:
     for filetype in ["svg", "png", "gif", "jpg", "ico"]:
-        if os.path.isfile(pathslug + "." + filetype):
+        if pathslug.with_suffix(pathslug.suffix + f".{filetype}").exists():
             return True
     return False
 
 
-def test_if_all_icons_are_converted(ignored_source_files):
+def walk(path: Path) -> Iterator[Path]:
+    for p in path.iterdir():
+        if p.is_dir():
+            yield p
+            yield from walk(p)
+            continue
+        yield p.resolve()
+
+
+def test_if_all_icons_are_converted(ignored_source_files) -> None:
     global error
     for filetype in ["svg", "png", "gif", "jpg", "ico"]:
-        for file in glob("src/**/*.{}".format(filetype)):
-            abs_dirname, filename = os.path.split(file)
-            code = os.path.splitext(filename)[0]
-            distfolder = "dist/" + abs_dirname[4:]
-            distfile = "{folder}/{code}.png".format(folder=distfolder, code=code)
+        for file in src.glob(f"**/*.{filetype}"):
+            distfile = Path("dist/") / Path(*file.parts[1:]).with_suffix(".png")
 
-            if not os.path.isfile(distfile) and file not in ignored_source_files:
-                print_error("{file} is missing (From {source})".format(file=distfile, source=file))
+            if not distfile.exists() and file not in ignored_source_files:
+                print_error(f"{distfile} is missing (From {file})")
                 error = True
 
-    return True
 
-
-def test_if_source_for_images():
+def test_if_source_for_images() -> None:
     global error
     for icontype in ["brand", "browsers", "os", "plugins", "SEO"]:
         for filetype in ["svg", "png", "gif", "jpg", "ico"]:
-            for source_file in glob("src/{type}/*.{filetype}".format(type=icontype, filetype=filetype)):
-                if not os.path.islink(source_file):
-                    if not os.path.isfile(source_file + ".source") and "UNK" not in source_file:
-                        print_error("Source is missing for {file}".format(file=source_file))
-                        error = True
+            for source_file in (src / icontype).glob(f"*.{filetype}"):
+                if (
+                        not source_file.is_symlink()
+                        and not source_file.with_suffix(source_file.suffix + ".source")
+                        and "UNK" not in source_file
+                ):
+                    print_error(f"Source is missing for {source_file}")
+                    error = True
 
 
-def test_if_all_symlinks_are_valid():
+def test_if_all_symlinks_are_valid() -> None:
     global error
-    for file in glob("src/**/*"):
-        if os.path.islink(file) and not os.path.exists(file):
-            print_error(
-                "Symlink doesn't link to file (from {link} to {target}".format(link=file, target=os.readlink(file))
-            )
+    for file in src.glob("**/*"):
+        if file.is_symlink():
+            target = file.resolve()
+            if not target.exists():
+                print_error(f"Symlink doesn't link to file (from {file} to {target} ({file.readlink()}))")
             error = True
 
 
-def test_if_placeholder_icon_exist(placeholder_icon_filenames):
+def test_if_placeholder_icon_exist(placeholder_icon_filenames: Dict[str, str]) -> None:
     global error
     for folder, filename in placeholder_icon_filenames.items():
-        file = "src/{folder}/{filename}".format(folder=folder, filename=filename)
-        if not (os.path.isfile(file) and hashlib.sha256(open(file, "rb").read()).hexdigest() == placeholder_icon_hash):
-            print_error("The placeholder icon {path} is missing or invalid".format(path=file))
+        file = src / folder / filename
+        if not (file.exists() and hashlib.sha256(file.read_bytes()).hexdigest() == placeholder_icon_hash):
+            print_error(f"The placeholder icon {file} is missing or invalid")
             error = True
 
 
-def test_if_icons_are_large_enough():
+def test_if_icons_are_large_enough() -> None:
     # ignore searchEngines and socials
     for filetype in ["png", "gif", "jpg", "ico"]:
-        for source_file in glob("src/*/*.{filetype}".format(filetype=filetype)):
+        for source_file in src.glob(f"*/*.{filetype}"):
             im = Image.open(source_file)
             if im.size[0] < min_image_size or im.size[1] < min_image_size:
+                width, height = im.size
                 print_warning(
-                    "{file} is smaller ({width}x{height}) that the target size ({target}x{target})".format(
-                        file=source_file,
-                        width=im.size[0],
-                        height=im.size[1],
-                        target=min_image_size
-                    )
+                    f"{source_file} is smaller ({width}x{height}) that the target size ({min_image_size}x{min_image_size})"
                 )
             if filetype in ["jpg", "gif", "ico"]:
-                print_warning("{file} is saved in a lossy image format ({filetype}). ".format(
-                    file=source_file,
-                    filetype=filetype
-                ) + "Maybe try to find an PNG or SVG from another source.")
+                print_warning(
+                    f"{source_file} is saved in a lossy image format ({filetype}). "
+                    "Maybe try to find an PNG or SVG from another source."
+                )
 
 
-def test_if_dist_icons_are_square(ignore_that_icon_isnt_square):
+def test_if_dist_icons_are_square(ignore_that_icon_isnt_square: List[str]) -> None:
     global error
-    for file in glob("dist/**/*.png"):
-        if "flags" not in file:
+    for file in dist.glob("**/*.png"):
+        if (dist / "flags") not in file.parents:
             im = Image.open(file)
-            if im.size[0] != im.size[1]:
-                string = "{file} isn't square ({width}x{height})".format(file=file, width=im.size[0], height=im.size[1])
+            width, height = im.size
+            if width != height:
+                string = f"{file} isn't square ({width}x{height})"
                 if file not in ignore_that_icon_isnt_square:
                     error = True
                     print_error(string)
@@ -135,36 +144,53 @@ def test_if_dist_icons_are_square(ignore_that_icon_isnt_square):
                     print_warning(string)
 
 
-def test_if_build_script_is_deleting_all_unneeded_files():
-    with open("tmp/piwik-package/scripts/build-package.sh") as f:
-        build_script = f.read()
+def is_in_allowed_dir(file: Path) -> bool:
+    allowed_dirs = [dist, Path("node_modules"), Path("tmp"), Path("vendor"), Path(".idea")]
+    for dir in allowed_dirs:
+        dir = dir.resolve()
+        if dir == file:
+            return True
+        if dir in file.parents:
+            return True
+    return False
+
+
+def is_deleted(file: Path, deleted_files: Iterable[Path]) -> bool:
+    # print(file, deleted_files)
+    for del_file in deleted_files:
+        if del_file == file:
+            return True
+        if del_file in file.parents:
+            return True
+    return False
+
+
+def test_if_build_script_is_deleting_all_unneeded_files() -> None:
     global error
-    deleted_files = []
-    all_files = []
-    for root, dirs, files in os.walk(".", topdown=False):
-        for name in files:
-            all_files.append(os.path.join(root, name))
-        for name in dirs:
-            all_files.append(os.path.join(root, name))
+    build_script = build_script_file.read_text()
+    deleted_files = set()
+    all_files = set(walk(Path(".").resolve()))
     for pattern in build_script_regex.findall(build_script):
-        deleted_files.extend(glob(pattern))
+        deleted_files.update(Path(".").resolve().glob(pattern))
     for file in all_files:
-        if not any(s in file for s in deleted_files) and not (
-                file.startswith("./dist") or file.startswith("./tmp") or file.startswith("./vendor")
-                or file.startswith("./node_modules")
-        ) and file != "./README.md":
-            print_error("{file} should be deleted by the build script".format(file=file))
-            error = True
+        if is_deleted(file, deleted_files):
+            continue
+        if is_in_allowed_dir(file):
+            continue
+        if file == Path("README.md").resolve():
+            continue
+        print_error(f"{file} should be deleted by the build script")
+        error = True
 
 
-def test_if_icons_are_indicated_to_be_improvable():
-    for file in glob("src/**/*.todo"):
-        print_warning("{icon} could be improved".format(icon=file[:-5]))
+def test_if_icons_are_indicated_to_be_improvable() -> None:
+    for file in src.glob("**/*.todo"):
+        print_warning(f"{str(file)[:-5]} could be improved")
 
 
-def look_for_search_and_social_icon(source, mode, outputdir):
+def look_for_search_and_social_icon(source, mode, outputdir: Path) -> None:
     global error
-    correct_files = []
+    correct_files = set()
     for i, element in source.items():
         if mode == "searchengines":
             search_engine = element[0]
@@ -173,32 +199,30 @@ def look_for_search_and_social_icon(source, mode, outputdir):
             urls = element
         url = next((url for url in urls if "{}" not in url), False)
         url = urlparse("https://" + url).netloc
-
-        if url and not image_exists(outputdir + url):
-            print_error("icon for {icon} is missing".format(icon=url))
+        if url and not image_exists(outputdir / url):
+            print_error(f"icon for {url} is missing")
             error = True
-        correct_files.append(url)
-        # print(correct_files)
+        correct_files.add(url)
     for filetype in ["svg", "png", "gif", "jpg", "ico"]:
-        for file in glob(outputdir + "*.{ext}".format(ext=filetype)):
-            domain = os.path.splitext(os.path.basename(file))[0]
+        for file in outputdir.glob(f"*.{filetype}"):
+            domain = file.stem
             if domain not in correct_files and domain != "xx":
-                print_error("{file} is not necessary".format(file=file))
+                print_error(f"{file} is not necessary")
                 error = True
 
 
-def test_if_all_search_and_social_sites_have_an_icon():
-    look_for_search_and_social_icon(load_yaml(searchEnginesFile), "searchengines", "src/searchEngines/")
-    look_for_search_and_social_icon(load_yaml(socialsEnginesFile), "socials", "src/socials/")
+def test_if_all_search_and_social_sites_have_an_icon() -> None:
+    look_for_search_and_social_icon(load_yaml(searchEnginesFile), "searchengines", Path("src/searchEngines/"))
+    look_for_search_and_social_icon(load_yaml(socialsEnginesFile), "socials", Path("src/socials/"))
 
 
-def test_if_there_are_icons_for_all_device_detector_categories(less_important_device_detector_icons):
+def test_if_there_are_icons_for_all_device_detector_categories(
+        less_important_device_detector_icons: Dict[str, List[str]]
+) -> None:
     global error
-    process = Popen(["php", "devicedetector.php"], stdout=PIPE)
-    (output, err) = process.communicate()
-    process.wait()
-    regex = re.compile(r"[^a-z0-9_\-]+",re.IGNORECASE)
-    categories = json.loads(output)
+    output = subprocess.run(["php", "devicedetector.php"], capture_output=True)
+    regex = re.compile(r"[^a-z0-9_\-]+", re.IGNORECASE)
+    categories = json.loads(output.stdout)
     for icontype, category in categories.items():
         for code in category:
             if icontype == "brand":
@@ -207,12 +231,12 @@ def test_if_there_are_icons_for_all_device_detector_categories(less_important_de
                 slug = code
             found = False
             for filetype in ["svg", "png", "gif", "jpg", "ico"]:
-                if os.path.isfile("src/{type}/{slug}.{ext}".format(type=icontype, slug=slug, ext=filetype)):
+                file = Path(f"src/{icontype}/{slug}.{filetype}")
+                if file.exists():
                     found = True
+                    break
             if not found:
-                warning = "icon for {icon} missing (should be at src/{type}/{slug}.{{png|svg}})".format(
-                    type=icontype, icon=category[code], slug=slug
-                )
+                warning = f"icon for {category[code]} missing (should be at src/{icontype}/{slug}.{{png|svg}})"
                 if slug in less_important_device_detector_icons[icontype]:
                     print_warning(warning)
                 else:
@@ -223,29 +247,26 @@ def test_if_there_are_icons_for_all_device_detector_categories(less_important_de
 if __name__ == "__main__":
     error = False
 
-    ignore = load_yaml("tests-ignore.yml")
+    ignore = load_yaml(Path("tests-ignore.yml"))
 
     if "TRAVIS_PULL_REQUEST" not in os.environ or not os.environ["TRAVIS_PULL_REQUEST"]:
         test_if_all_icons_are_converted(ignore["ignored_source_files"])
+    exit()
 
     test_if_source_for_images()
     test_if_all_symlinks_are_valid()
     test_if_placeholder_icon_exist(ignore["placeholder_icon_filenames"])
     test_if_dist_icons_are_square(ignore["ignore_that_icon_isnt_square"])
-    if "TRAVIS" in os.environ and os.environ["TRAVIS"]:  # collapse on travis
+    travis = "TRAVIS" in os.environ and os.environ["TRAVIS"]  # collapse on travis
+    if travis:
         print("travis_fold:start:improvable_icons")
         print("improvable icons: (click to expand)")
-        test_if_there_are_icons_for_all_device_detector_categories(ignore["less_important_device_detector_icons"])
-        test_if_icons_are_indicated_to_be_improvable()
-        test_if_icons_are_large_enough()
+    test_if_there_are_icons_for_all_device_detector_categories(ignore["less_important_device_detector_icons"])
+    test_if_icons_are_indicated_to_be_improvable()
+    test_if_icons_are_large_enough()
+    if travis:
         print("travis_fold:end:improvable_icons")
-        test_if_all_search_and_social_sites_have_an_icon()
-        test_if_build_script_is_deleting_all_unneeded_files()
-    else:
-        test_if_there_are_icons_for_all_device_detector_categories(ignore["less_important_device_detector_icons"])
-        test_if_icons_are_indicated_to_be_improvable()
-        test_if_icons_are_large_enough()
-        test_if_all_search_and_social_sites_have_an_icon()
-        test_if_build_script_is_deleting_all_unneeded_files()
+    test_if_all_search_and_social_sites_have_an_icon()
+    test_if_build_script_is_deleting_all_unneeded_files()
 
     sys.exit(error)