From d8b477df08eb1d269de1ec22368d7b761d94f0ae Mon Sep 17 00:00:00 2001 From: Lukas Winkler Date: Sat, 19 Feb 2022 21:14:08 +0100 Subject: [PATCH] rewrite test script --- .travis.yml | 2 +- requirements.txt | 4 +- tests.py | 239 ++++++++++++++++++++++++++--------------------- 3 files changed, 133 insertions(+), 112 deletions(-) diff --git a/.travis.yml b/.travis.yml index 786424c..8051e57 100644 --- a/.travis.yml +++ b/.travis.yml @@ -2,7 +2,7 @@ os: linux dist: bionic language: python python: - "3.8" + "3.9" before_install: - git clone https://github.com/piwik/piwik-package.git tmp/piwik-package diff --git a/requirements.txt b/requirements.txt index 9721d8c..87c0242 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,2 +1,2 @@ -Pillow==8.1.2 -PyYAML==5.4.1 +Pillow==9.0.1 +PyYAML==6.0 diff --git a/tests.py b/tests.py index fe3b34b..ff27ea7 100755 --- a/tests.py +++ b/tests.py @@ -16,9 +16,10 @@ import hashlib import json import os import re +import subprocess import sys -from glob import glob -from subprocess import Popen, PIPE +from pathlib import Path +from typing import Dict, List, Iterator, Iterable from urllib.parse import urlparse import yaml @@ -30,104 +31,112 @@ min_image_size = 48 placeholder_icon_hash = "398a623a3b0b10eba6d1884b0ff1713ee12aeafaa8efaf67b60a4624f4dce48c" -searchEnginesFile = "vendor/matomo/searchengine-and-social-list/SearchEngines.yml" -socialsEnginesFile = "vendor/matomo/searchengine-and-social-list/Socials.yml" +searchEnginesFile = Path("vendor/matomo/searchengine-and-social-list/SearchEngines.yml") +socialsEnginesFile = Path("vendor/matomo/searchengine-and-social-list/Socials.yml") +build_script_file = Path("tmp/piwik-package/scripts/build-package.sh") + +src = Path("src/") +dist: Path = Path("dist/") -def print_warning(string): +def print_warning(string: str) -> None: print("\033[33;1m⚠\033[0m " + string) -def print_error(string): +def print_error(string: str) -> None: print("\033[31;1m⚠ " + string + "\033[0m") -def load_yaml(file): - with open(file, 'r') as stream: +def load_yaml(file: Path): + with file.open() as stream: return yaml.safe_load(stream) -def image_exists(pathslug): +def image_exists(pathslug: Path) -> bool: for filetype in ["svg", "png", "gif", "jpg", "ico"]: - if os.path.isfile(pathslug + "." + filetype): + if pathslug.with_suffix(pathslug.suffix + f".{filetype}").exists(): return True return False -def test_if_all_icons_are_converted(ignored_source_files): +def walk(path: Path) -> Iterator[Path]: + for p in path.iterdir(): + if p.is_dir(): + yield p + yield from walk(p) + continue + yield p.resolve() + + +def test_if_all_icons_are_converted(ignored_source_files) -> None: global error for filetype in ["svg", "png", "gif", "jpg", "ico"]: - for file in glob("src/**/*.{}".format(filetype)): - abs_dirname, filename = os.path.split(file) - code = os.path.splitext(filename)[0] - distfolder = "dist/" + abs_dirname[4:] - distfile = "{folder}/{code}.png".format(folder=distfolder, code=code) + for file in src.glob(f"**/*.{filetype}"): + distfile = Path("dist/") / Path(*file.parts[1:]).with_suffix(".png") - if not os.path.isfile(distfile) and file not in ignored_source_files: - print_error("{file} is missing (From {source})".format(file=distfile, source=file)) + if not distfile.exists() and file not in ignored_source_files: + print_error(f"{distfile} is missing (From {file})") error = True - return True - -def test_if_source_for_images(): +def test_if_source_for_images() -> None: global error for icontype in ["brand", "browsers", "os", "plugins", "SEO"]: for filetype in ["svg", "png", "gif", "jpg", "ico"]: - for source_file in glob("src/{type}/*.{filetype}".format(type=icontype, filetype=filetype)): - if not os.path.islink(source_file): - if not os.path.isfile(source_file + ".source") and "UNK" not in source_file: - print_error("Source is missing for {file}".format(file=source_file)) - error = True + for source_file in (src / icontype).glob(f"*.{filetype}"): + if ( + not source_file.is_symlink() + and not source_file.with_suffix(source_file.suffix + ".source") + and "UNK" not in source_file + ): + print_error(f"Source is missing for {source_file}") + error = True -def test_if_all_symlinks_are_valid(): +def test_if_all_symlinks_are_valid() -> None: global error - for file in glob("src/**/*"): - if os.path.islink(file) and not os.path.exists(file): - print_error( - "Symlink doesn't link to file (from {link} to {target}".format(link=file, target=os.readlink(file)) - ) + for file in src.glob("**/*"): + if file.is_symlink(): + target = file.resolve() + if not target.exists(): + print_error(f"Symlink doesn't link to file (from {file} to {target} ({file.readlink()}))") error = True -def test_if_placeholder_icon_exist(placeholder_icon_filenames): +def test_if_placeholder_icon_exist(placeholder_icon_filenames: Dict[str, str]) -> None: global error for folder, filename in placeholder_icon_filenames.items(): - file = "src/{folder}/{filename}".format(folder=folder, filename=filename) - if not (os.path.isfile(file) and hashlib.sha256(open(file, "rb").read()).hexdigest() == placeholder_icon_hash): - print_error("The placeholder icon {path} is missing or invalid".format(path=file)) + file = src / folder / filename + if not (file.exists() and hashlib.sha256(file.read_bytes()).hexdigest() == placeholder_icon_hash): + print_error(f"The placeholder icon {file} is missing or invalid") error = True -def test_if_icons_are_large_enough(): +def test_if_icons_are_large_enough() -> None: # ignore searchEngines and socials for filetype in ["png", "gif", "jpg", "ico"]: - for source_file in glob("src/*/*.{filetype}".format(filetype=filetype)): + for source_file in src.glob(f"*/*.{filetype}"): im = Image.open(source_file) if im.size[0] < min_image_size or im.size[1] < min_image_size: + width, height = im.size print_warning( - "{file} is smaller ({width}x{height}) that the target size ({target}x{target})".format( - file=source_file, - width=im.size[0], - height=im.size[1], - target=min_image_size - ) + f"{source_file} is smaller ({width}x{height}) that the target size ({min_image_size}x{min_image_size})" ) if filetype in ["jpg", "gif", "ico"]: - print_warning("{file} is saved in a lossy image format ({filetype}). ".format( - file=source_file, - filetype=filetype - ) + "Maybe try to find an PNG or SVG from another source.") + print_warning( + f"{source_file} is saved in a lossy image format ({filetype}). " + "Maybe try to find an PNG or SVG from another source." + ) -def test_if_dist_icons_are_square(ignore_that_icon_isnt_square): +def test_if_dist_icons_are_square(ignore_that_icon_isnt_square: List[str]) -> None: global error - for file in glob("dist/**/*.png"): - if "flags" not in file: + for file in dist.glob("**/*.png"): + if (dist / "flags") not in file.parents: im = Image.open(file) - if im.size[0] != im.size[1]: - string = "{file} isn't square ({width}x{height})".format(file=file, width=im.size[0], height=im.size[1]) + width, height = im.size + if width != height: + string = f"{file} isn't square ({width}x{height})" if file not in ignore_that_icon_isnt_square: error = True print_error(string) @@ -135,36 +144,53 @@ def test_if_dist_icons_are_square(ignore_that_icon_isnt_square): print_warning(string) -def test_if_build_script_is_deleting_all_unneeded_files(): - with open("tmp/piwik-package/scripts/build-package.sh") as f: - build_script = f.read() +def is_in_allowed_dir(file: Path) -> bool: + allowed_dirs = [dist, Path("node_modules"), Path("tmp"), Path("vendor"), Path(".idea")] + for dir in allowed_dirs: + dir = dir.resolve() + if dir == file: + return True + if dir in file.parents: + return True + return False + + +def is_deleted(file: Path, deleted_files: Iterable[Path]) -> bool: + # print(file, deleted_files) + for del_file in deleted_files: + if del_file == file: + return True + if del_file in file.parents: + return True + return False + + +def test_if_build_script_is_deleting_all_unneeded_files() -> None: global error - deleted_files = [] - all_files = [] - for root, dirs, files in os.walk(".", topdown=False): - for name in files: - all_files.append(os.path.join(root, name)) - for name in dirs: - all_files.append(os.path.join(root, name)) + build_script = build_script_file.read_text() + deleted_files = set() + all_files = set(walk(Path(".").resolve())) for pattern in build_script_regex.findall(build_script): - deleted_files.extend(glob(pattern)) + deleted_files.update(Path(".").resolve().glob(pattern)) for file in all_files: - if not any(s in file for s in deleted_files) and not ( - file.startswith("./dist") or file.startswith("./tmp") or file.startswith("./vendor") - or file.startswith("./node_modules") - ) and file != "./README.md": - print_error("{file} should be deleted by the build script".format(file=file)) - error = True + if is_deleted(file, deleted_files): + continue + if is_in_allowed_dir(file): + continue + if file == Path("README.md").resolve(): + continue + print_error(f"{file} should be deleted by the build script") + error = True -def test_if_icons_are_indicated_to_be_improvable(): - for file in glob("src/**/*.todo"): - print_warning("{icon} could be improved".format(icon=file[:-5])) +def test_if_icons_are_indicated_to_be_improvable() -> None: + for file in src.glob("**/*.todo"): + print_warning(f"{str(file)[:-5]} could be improved") -def look_for_search_and_social_icon(source, mode, outputdir): +def look_for_search_and_social_icon(source, mode, outputdir: Path) -> None: global error - correct_files = [] + correct_files = set() for i, element in source.items(): if mode == "searchengines": search_engine = element[0] @@ -173,32 +199,30 @@ def look_for_search_and_social_icon(source, mode, outputdir): urls = element url = next((url for url in urls if "{}" not in url), False) url = urlparse("https://" + url).netloc - - if url and not image_exists(outputdir + url): - print_error("icon for {icon} is missing".format(icon=url)) + if url and not image_exists(outputdir / url): + print_error(f"icon for {url} is missing") error = True - correct_files.append(url) - # print(correct_files) + correct_files.add(url) for filetype in ["svg", "png", "gif", "jpg", "ico"]: - for file in glob(outputdir + "*.{ext}".format(ext=filetype)): - domain = os.path.splitext(os.path.basename(file))[0] + for file in outputdir.glob(f"*.{filetype}"): + domain = file.stem if domain not in correct_files and domain != "xx": - print_error("{file} is not necessary".format(file=file)) + print_error(f"{file} is not necessary") error = True -def test_if_all_search_and_social_sites_have_an_icon(): - look_for_search_and_social_icon(load_yaml(searchEnginesFile), "searchengines", "src/searchEngines/") - look_for_search_and_social_icon(load_yaml(socialsEnginesFile), "socials", "src/socials/") +def test_if_all_search_and_social_sites_have_an_icon() -> None: + look_for_search_and_social_icon(load_yaml(searchEnginesFile), "searchengines", Path("src/searchEngines/")) + look_for_search_and_social_icon(load_yaml(socialsEnginesFile), "socials", Path("src/socials/")) -def test_if_there_are_icons_for_all_device_detector_categories(less_important_device_detector_icons): +def test_if_there_are_icons_for_all_device_detector_categories( + less_important_device_detector_icons: Dict[str, List[str]] +) -> None: global error - process = Popen(["php", "devicedetector.php"], stdout=PIPE) - (output, err) = process.communicate() - process.wait() - regex = re.compile(r"[^a-z0-9_\-]+",re.IGNORECASE) - categories = json.loads(output) + output = subprocess.run(["php", "devicedetector.php"], capture_output=True) + regex = re.compile(r"[^a-z0-9_\-]+", re.IGNORECASE) + categories = json.loads(output.stdout) for icontype, category in categories.items(): for code in category: if icontype == "brand": @@ -207,12 +231,12 @@ def test_if_there_are_icons_for_all_device_detector_categories(less_important_de slug = code found = False for filetype in ["svg", "png", "gif", "jpg", "ico"]: - if os.path.isfile("src/{type}/{slug}.{ext}".format(type=icontype, slug=slug, ext=filetype)): + file = Path(f"src/{icontype}/{slug}.{filetype}") + if file.exists(): found = True + break if not found: - warning = "icon for {icon} missing (should be at src/{type}/{slug}.{{png|svg}})".format( - type=icontype, icon=category[code], slug=slug - ) + warning = f"icon for {category[code]} missing (should be at src/{icontype}/{slug}.{{png|svg}})" if slug in less_important_device_detector_icons[icontype]: print_warning(warning) else: @@ -223,29 +247,26 @@ def test_if_there_are_icons_for_all_device_detector_categories(less_important_de if __name__ == "__main__": error = False - ignore = load_yaml("tests-ignore.yml") + ignore = load_yaml(Path("tests-ignore.yml")) if "TRAVIS_PULL_REQUEST" not in os.environ or not os.environ["TRAVIS_PULL_REQUEST"]: test_if_all_icons_are_converted(ignore["ignored_source_files"]) + exit() test_if_source_for_images() test_if_all_symlinks_are_valid() test_if_placeholder_icon_exist(ignore["placeholder_icon_filenames"]) test_if_dist_icons_are_square(ignore["ignore_that_icon_isnt_square"]) - if "TRAVIS" in os.environ and os.environ["TRAVIS"]: # collapse on travis + travis = "TRAVIS" in os.environ and os.environ["TRAVIS"] # collapse on travis + if travis: print("travis_fold:start:improvable_icons") print("improvable icons: (click to expand)") - test_if_there_are_icons_for_all_device_detector_categories(ignore["less_important_device_detector_icons"]) - test_if_icons_are_indicated_to_be_improvable() - test_if_icons_are_large_enough() + test_if_there_are_icons_for_all_device_detector_categories(ignore["less_important_device_detector_icons"]) + test_if_icons_are_indicated_to_be_improvable() + test_if_icons_are_large_enough() + if travis: print("travis_fold:end:improvable_icons") - test_if_all_search_and_social_sites_have_an_icon() - test_if_build_script_is_deleting_all_unneeded_files() - else: - test_if_there_are_icons_for_all_device_detector_categories(ignore["less_important_device_detector_icons"]) - test_if_icons_are_indicated_to_be_improvable() - test_if_icons_are_large_enough() - test_if_all_search_and_social_sites_have_an_icon() - test_if_build_script_is_deleting_all_unneeded_files() + test_if_all_search_and_social_sites_have_an_icon() + test_if_build_script_is_deleting_all_unneeded_files() sys.exit(error)