mirror of
https://github.com/cosmo-sims/cosmICweb-music.git
synced 2024-09-19 16:53:43 +02:00
commit
92f9d78f12
8 changed files with 507 additions and 120 deletions
1
.gitignore
vendored
1
.gitignore
vendored
|
@ -1,3 +1,4 @@
|
|||
__pycache__
|
||||
.venv
|
||||
dist
|
||||
*.cfg
|
||||
|
|
|
@ -1,8 +1,11 @@
|
|||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import sys
|
||||
import tempfile
|
||||
import subprocess
|
||||
from typing import NamedTuple, Any, List, Dict
|
||||
from typing import Any
|
||||
from .data_types import Ellipsoid, Args, DownloadConfig
|
||||
|
||||
import click
|
||||
import requests
|
||||
|
@ -22,35 +25,10 @@ logger.setLevel("INFO")
|
|||
# Some constants
|
||||
DEFAULT_URL = "https://cosmicweb.eu"
|
||||
EDITOR = os.environ.get("EDITOR", "vim")
|
||||
EDITOR_IS_VIM = EDITOR in {"vim", "nvim"}
|
||||
|
||||
|
||||
# Types
|
||||
class Ellipsoid(NamedTuple):
|
||||
center: int
|
||||
shape: int
|
||||
traceback_radius: int
|
||||
radius_definition: int
|
||||
|
||||
|
||||
class DownloadConfig(NamedTuple):
|
||||
simulation_name: str
|
||||
halo_names: List[Any]
|
||||
halo_urls: List[str]
|
||||
traceback_radius: float
|
||||
api_token: str
|
||||
MUSIC: str
|
||||
settings: Dict[Any, Any]
|
||||
accessed_at: datetime
|
||||
|
||||
|
||||
class Args(NamedTuple):
|
||||
url: str
|
||||
output_path: str
|
||||
common_directory: str
|
||||
attempts: int
|
||||
|
||||
|
||||
def query_yes_no(question, default="yes"):
|
||||
def query_yes_no(question: str, default="yes") -> bool:
|
||||
"""Ask a yes/no question via raw_input() and return their answer.
|
||||
|
||||
"question" is a string that is presented to the user.
|
||||
|
@ -82,7 +60,7 @@ def query_yes_no(question, default="yes"):
|
|||
|
||||
|
||||
# Routines
|
||||
def fetch_ellipsoids(url, api_token, attempts):
|
||||
def fetch_ellipsoids(url: str, api_token: str, attempts: int) -> list[Ellipsoid]:
|
||||
for i in range(attempts):
|
||||
try:
|
||||
r = requests.get(url, headers={"Authorization": "Token " + api_token})
|
||||
|
@ -103,19 +81,21 @@ def fetch_ellipsoids(url, api_token, attempts):
|
|||
for e in content
|
||||
]
|
||||
logging.error("Unable to download ellipsoids from {}".format(url))
|
||||
return None
|
||||
return []
|
||||
|
||||
|
||||
def fetch_ellipsoid(url, api_token, traceback_radius, attempts=3):
|
||||
def fetch_ellipsoid(
|
||||
url: str, api_token: str, traceback_radius:float, attempts: int = 3
|
||||
) -> Ellipsoid | None:
|
||||
ellipsoids = fetch_ellipsoids(url, api_token, attempts)
|
||||
if ellipsoids is not None:
|
||||
if ellipsoids:
|
||||
return next(
|
||||
(e for e in ellipsoids if e.traceback_radius == traceback_radius), None
|
||||
)
|
||||
return None
|
||||
|
||||
|
||||
def fetch_downloadstore(cosmicweb_url, target):
|
||||
def fetch_downloadstore(cosmicweb_url: str, target: str) -> DownloadConfig:
|
||||
try:
|
||||
r = requests.get(cosmicweb_url + "/api/music/store/" + target)
|
||||
# This will raise an error if not successful
|
||||
|
@ -134,7 +114,9 @@ def fetch_downloadstore(cosmicweb_url, target):
|
|||
]
|
||||
return DownloadConfig(
|
||||
simulation_name=sim["name"],
|
||||
project_name=sim["project_name"],
|
||||
halo_names=["halo_{}".format(h) for h in content["halos"]],
|
||||
halo_ids=content["halos"],
|
||||
halo_urls=halo_urls,
|
||||
traceback_radius=content["traceback_radius"],
|
||||
api_token=sim["api_token"],
|
||||
|
@ -144,7 +126,9 @@ def fetch_downloadstore(cosmicweb_url, target):
|
|||
)
|
||||
|
||||
|
||||
def fetch_publication(cosmicweb_url, publication_name, traceback_radius):
|
||||
def fetch_publication(
|
||||
cosmicweb_url: str, publication_name: str, traceback_radius
|
||||
) -> DownloadConfig:
|
||||
try:
|
||||
r = requests.get(cosmicweb_url + "/api/publications/" + publication_name)
|
||||
# This will raise an error if not successful
|
||||
|
@ -156,6 +140,7 @@ def fetch_publication(cosmicweb_url, publication_name, traceback_radius):
|
|||
content = r.json()
|
||||
sim = content["simulation"]
|
||||
halo_names = [h["name"] for h in content["halos"]]
|
||||
halo_ids = [h["id"] for h in content["halos"]]
|
||||
halo_urls = [
|
||||
"{url}/simulation/{sid}/halo/{hid}".format(
|
||||
url=sim["api_url"], sid=sim["api_id"], hid=h["id"]
|
||||
|
@ -164,38 +149,73 @@ def fetch_publication(cosmicweb_url, publication_name, traceback_radius):
|
|||
]
|
||||
return DownloadConfig(
|
||||
simulation_name=sim["name"],
|
||||
project_name=sim["project_name"],
|
||||
halo_names=halo_names,
|
||||
halo_ids=halo_ids,
|
||||
halo_urls=halo_urls,
|
||||
traceback_radius=traceback_radius,
|
||||
api_token=sim["api_token"],
|
||||
MUSIC=sim["ics"],
|
||||
settings={},
|
||||
settings=None,
|
||||
accessed_at=datetime.now(),
|
||||
)
|
||||
|
||||
|
||||
def edit_template(template):
|
||||
with tempfile.NamedTemporaryFile(suffix=".tmp", mode="r+") as tf:
|
||||
def edit_template(template: str) -> str:
|
||||
with tempfile.NamedTemporaryFile(suffix=".tmp.conf", mode="r+") as tf:
|
||||
tf.write(template)
|
||||
tf.flush()
|
||||
# Call the editor. backupcopy=yes prevents vim from creating copy and rename
|
||||
subprocess.call([EDITOR, "+set backupcopy=yes", tf.name])
|
||||
editor_parameters = []
|
||||
if EDITOR_IS_VIM:
|
||||
# backupcopy=yes prevents vim from creating copy and rename
|
||||
editor_parameters.append("+set backupcopy=yes")
|
||||
subprocess.call([EDITOR] + editor_parameters + [tf.name])
|
||||
tf.seek(0)
|
||||
template = tf.read()
|
||||
return template
|
||||
|
||||
|
||||
def music_config_to_template(music_config, configuration):
|
||||
# TODO: apply configuraton, add header
|
||||
return (
|
||||
def apply_config_parameter(config: str, parameters: dict[str, Any]) -> str:
|
||||
new_lines = []
|
||||
for line in config.split("\n"):
|
||||
param = line.split("=")[0].strip()
|
||||
if param in parameters:
|
||||
line = line.split("=")[0] + f"= {parameters[param]}"
|
||||
new_lines.append(line)
|
||||
return "\n".join(new_lines)
|
||||
|
||||
|
||||
def music_config_to_template(config: DownloadConfig) -> str:
|
||||
music_config = config.MUSIC
|
||||
settings = config.settings
|
||||
# TODO: apply output configuration
|
||||
config = (
|
||||
"[setup]\n" + music_config["setup"] + "\n\n<ELLIPSOID_TEMPLATE>\n\n"
|
||||
"[cosmology]\n" + music_config["cosmology"] + "\n\n"
|
||||
"[random]\n" + music_config["random"] + "\n\n"
|
||||
"[poisson]\n" + music_config["poisson"]
|
||||
)
|
||||
if settings:
|
||||
config = apply_config_parameter(
|
||||
config,
|
||||
{
|
||||
"levelmin": settings["resolution"]["low"],
|
||||
"levelmin_TF": settings["resolution"]["low"],
|
||||
"levelmax": settings["resolution"]["high"],
|
||||
"zstart": settings["startRedshift"],
|
||||
},
|
||||
)
|
||||
return config
|
||||
|
||||
|
||||
def compose_template(template, ellipsoid):
|
||||
def compose_template(
|
||||
template: str,
|
||||
ellipsoid: Ellipsoid,
|
||||
config: DownloadConfig,
|
||||
halo_name: str,
|
||||
halo_id: int,
|
||||
now: datetime = None,
|
||||
) -> str:
|
||||
# TODO: add ellipsoid header (rtb, halo_name, etc)
|
||||
shape_0 = ", ".join(str(e) for e in ellipsoid.shape[0])
|
||||
shape_1 = ", ".join(str(e) for e in ellipsoid.shape[1])
|
||||
|
@ -203,18 +223,27 @@ def compose_template(template, ellipsoid):
|
|||
center = ", ".join(str(x) for x in ellipsoid.center)
|
||||
|
||||
ellipsoid_lines = (
|
||||
"# Ellipsoidal refinement region defined on unity cube\n"
|
||||
"# This minimum bounding ellipsoid has been obtained from\n"
|
||||
f"# particles within {ellipsoid.traceback_radius} {ellipsoid.radius_definition} of the halo center\n"
|
||||
"region = ellipsoid\n"
|
||||
"region_ellipsoid_matrix[0] = {s0}\n"
|
||||
"region_ellipsoid_matrix[1] = {s1}\n"
|
||||
"region_ellipsoid_matrix[2] = {s2}\n"
|
||||
"region_ellipsoid_center = {c}\n".format(
|
||||
s0=shape_0, s1=shape_1, s2=shape_2, c=center
|
||||
)
|
||||
f"region_ellipsoid_matrix[0] = {shape_0}\n"
|
||||
f"region_ellipsoid_matrix[1] = {shape_1}\n"
|
||||
f"region_ellipsoid_matrix[2] = {shape_2}\n"
|
||||
f"region_ellipsoid_center = {center}\n"
|
||||
)
|
||||
return template.replace("<ELLIPSOID_TEMPLATE>", ellipsoid_lines)
|
||||
template = template.replace("<ELLIPSOID_TEMPLATE>", ellipsoid_lines)
|
||||
if now is None:
|
||||
now = datetime.now()
|
||||
config_header = (
|
||||
f"# Zoom Initial Conditions for halo {halo_id} ({halo_name}) in simulation {config.simulation_name} ({config.project_name} project)\n"
|
||||
f"# Details on this halo can be found on https://cosmicweb.eu/simulation/{config.simulation_name}/halo/{halo_id}\n"
|
||||
f"# This file has been generated by CosmICweb @{now.isoformat()}\n\n\n"
|
||||
)
|
||||
return config_header + template + "\n"
|
||||
|
||||
|
||||
def write_music_file(output_file, music_config):
|
||||
def write_music_file(output_file: str, music_config: str) -> None:
|
||||
dirname = os.path.dirname(output_file)
|
||||
if not os.path.exists(dirname):
|
||||
logging.debug("Creating directory {}".format(dirname))
|
||||
|
@ -223,11 +252,11 @@ def write_music_file(output_file, music_config):
|
|||
f.write(music_config)
|
||||
|
||||
|
||||
def call_music():
|
||||
def call_music() -> None:
|
||||
pass
|
||||
|
||||
|
||||
def process_config(config, args: Args):
|
||||
def process_config(config: DownloadConfig, args: Args, store: bool) -> None:
|
||||
ellipsoids = []
|
||||
for halo_name, url in zip(config.halo_names, config.halo_urls):
|
||||
logging.info("Fetching ellipsoids from halo " + halo_name)
|
||||
|
@ -239,12 +268,12 @@ def process_config(config, args: Args):
|
|||
args.attempts,
|
||||
)
|
||||
)
|
||||
|
||||
# Edit template
|
||||
logging.info("Creating MUSIC template")
|
||||
music_template = music_config_to_template(config.MUSIC, config.settings)
|
||||
music_template = music_config_to_template(config)
|
||||
output = []
|
||||
|
||||
if query_yes_no(
|
||||
if store and query_yes_no(
|
||||
"Do you want to edit the MUSIC template before creating the IC files?\n"
|
||||
"(changing zstart, levelmin, levelmax, etc.)",
|
||||
default="no",
|
||||
|
@ -253,41 +282,48 @@ def process_config(config, args: Args):
|
|||
music_template = edit_template(music_template)
|
||||
logging.debug("Finished editing MUSIC template")
|
||||
# Store template to file
|
||||
for halo_name, ellipsoid in zip(config.halo_names, ellipsoids):
|
||||
for halo_name, halo_id, ellipsoid in zip(
|
||||
config.halo_names, config.halo_ids, ellipsoids
|
||||
):
|
||||
if ellipsoid is None:
|
||||
logging.warning(
|
||||
"Ellipsoid for halo {} not available, skipping".format(halo_name)
|
||||
)
|
||||
continue
|
||||
logging.info("Composing MUSIC configuration file for halo {}".format(halo_name))
|
||||
music_config = compose_template(music_template, ellipsoid)
|
||||
music_config = compose_template(
|
||||
music_template, ellipsoid, config, halo_name, halo_id
|
||||
)
|
||||
if args.common_directory and len(ellipsoids) > 1:
|
||||
output_file = os.path.join(args.output_path, str(halo_name), "ics.cfg")
|
||||
else:
|
||||
output_file = os.path.join(
|
||||
args.output_path, "ics_{}.cfg".format(halo_name)
|
||||
)
|
||||
output_file = os.path.join(args.output_path, "ics_{}.cfg".format(halo_name))
|
||||
logging.info(
|
||||
"Storing MUSIC configuration file for halo {} in {}".format(
|
||||
halo_name, output_file
|
||||
)
|
||||
)
|
||||
write_music_file(output_file, music_config)
|
||||
|
||||
if store:
|
||||
write_music_file(output_file, music_config)
|
||||
else:
|
||||
output.append((output_file, music_config))
|
||||
return output
|
||||
# TODO: Execute MUSIC?
|
||||
|
||||
|
||||
def downloadstore_mode(args: Args, target: str):
|
||||
def downloadstore_mode(args: Args, target: str, store=True) -> None | str:
|
||||
logging.info("Fetching download configuration from the cosmICweb server")
|
||||
config = fetch_downloadstore(args.url, target)
|
||||
if args.output_path == "./":
|
||||
args = args._replace(output_path=f"./cosmICweb-zooms-{config.simulation_name}")
|
||||
logging.debug("Output directory set to " + args.output_path)
|
||||
logging.info("Download configuration successfully fetched")
|
||||
process_config(config, args)
|
||||
return process_config(config, args, store)
|
||||
|
||||
|
||||
def publication_mode(args: Args, publication_name: str, traceback_radius: int):
|
||||
def publication_mode(
|
||||
args: Args, publication_name: str, traceback_radius, store=True
|
||||
) -> None | str:
|
||||
logging.info(
|
||||
"Fetching publication " + publication_name + " from the cosmICweb server"
|
||||
)
|
||||
|
@ -295,10 +331,10 @@ def publication_mode(args: Args, publication_name: str, traceback_radius: int):
|
|||
args = args._replace(output_path=os.path.join(args.output_path, publication_name))
|
||||
logging.debug("Output directory set to " + args.output_path)
|
||||
logging.info("Publication successfully fetched")
|
||||
process_config(config, args)
|
||||
return process_config(config, args, store)
|
||||
|
||||
|
||||
def dir_path(p):
|
||||
def dir_path(p: str) -> str:
|
||||
if os.path.isdir(p):
|
||||
return p
|
||||
else:
|
||||
|
@ -351,54 +387,3 @@ def publication(ctx, publication_name, traceback_radius):
|
|||
|
||||
if __name__ == "__main__":
|
||||
cli()
|
||||
|
||||
# if __name__ == "__main__":
|
||||
# parser = argparse.ArgumentParser()
|
||||
# parser.add_argument(
|
||||
# "--url",
|
||||
# dest="cosmicweb_url",
|
||||
# default=DEFAULT_URL,
|
||||
# help="overwrite URL of the cosmicweb server",
|
||||
# )
|
||||
# parser.add_argument(
|
||||
# "--output-path",
|
||||
# type=dir_path,
|
||||
# default="./",
|
||||
# help="Download target for IC files. If downloading publication, will create a subfolder with the "
|
||||
# "name of the publication",
|
||||
# )
|
||||
# parser.add_argument(
|
||||
# "--common-directory", dest="create_subdirs", action="store_false"
|
||||
# )
|
||||
# parser.add_argument(
|
||||
# "--attempts",
|
||||
# type=int,
|
||||
# default=3,
|
||||
# help="number of attempts to download ellipsoids",
|
||||
# )
|
||||
# parser.add_argument("--verbose", action="store_true")
|
||||
|
||||
# subparsers = parser.add_subparsers(dest="mode")
|
||||
# # Downloading from publications
|
||||
# publication_parser = subparsers.add_parser(
|
||||
# "publication", help="download publications"
|
||||
# )
|
||||
# publication_parser.add_argument("publication_name", help="name of the publication")
|
||||
# publication_parser.add_argument(
|
||||
# "--traceback_radius", type=int, choices=[1, 2, 4, 10], default=2, help=""
|
||||
# )
|
||||
# # Downloading from download object
|
||||
# download_parser = subparsers.add_parser("get")
|
||||
# download_parser.add_argument("target")
|
||||
|
||||
# args = parser.parse_args()
|
||||
|
||||
# if args.verbose:
|
||||
# logger.setLevel("DEBUG")
|
||||
|
||||
# if args.mode == "get":
|
||||
# downloadstore_mode(args)
|
||||
# elif args.mode == "publication":
|
||||
# publication_mode(args)
|
||||
# else:
|
||||
# raise NotImplementedError("unknown subparser")
|
||||
|
|
53
cosmicweb_music/data_types.py
Normal file
53
cosmicweb_music/data_types.py
Normal file
|
@ -0,0 +1,53 @@
|
|||
from __future__ import annotations
|
||||
|
||||
from datetime import datetime
|
||||
from typing import NamedTuple, Any, List, Dict, TypedDict
|
||||
|
||||
|
||||
class Ellipsoid(NamedTuple):
|
||||
center: List[float]
|
||||
shape: List[List[float]]
|
||||
traceback_radius: float
|
||||
radius_definition: str
|
||||
|
||||
|
||||
class Resolution(TypedDict):
|
||||
low: int
|
||||
high: int
|
||||
|
||||
|
||||
class Configuration(TypedDict):
|
||||
outputType: str
|
||||
resolution: Resolution
|
||||
outputOptions: List[Any]
|
||||
startRedshift: int
|
||||
outputFilename: str
|
||||
separateFolders: bool
|
||||
tracebackRadius: int | float | str
|
||||
|
||||
|
||||
class ICSections(TypedDict):
|
||||
setup: str
|
||||
random: str
|
||||
cosmology: str
|
||||
poisson: str
|
||||
|
||||
|
||||
class DownloadConfig(NamedTuple):
|
||||
simulation_name: str
|
||||
project_name: str
|
||||
halo_names: List[str]
|
||||
halo_ids: List[int]
|
||||
halo_urls: List[str]
|
||||
traceback_radius: float
|
||||
api_token: str
|
||||
MUSIC: ICSections
|
||||
settings: Configuration | None
|
||||
accessed_at: datetime
|
||||
|
||||
|
||||
class Args(NamedTuple):
|
||||
url: str
|
||||
output_path: str
|
||||
common_directory: bool
|
||||
attempts: int
|
86
poetry.lock
generated
86
poetry.lock
generated
|
@ -135,6 +135,20 @@ files = [
|
|||
{file = "colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44"},
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "exceptiongroup"
|
||||
version = "1.2.1"
|
||||
description = "Backport of PEP 654 (exception groups)"
|
||||
optional = false
|
||||
python-versions = ">=3.7"
|
||||
files = [
|
||||
{file = "exceptiongroup-1.2.1-py3-none-any.whl", hash = "sha256:5258b9ed329c5bbdd31a309f53cbfb0b155341807f6ff7606a1e801a891b29ad"},
|
||||
{file = "exceptiongroup-1.2.1.tar.gz", hash = "sha256:a4785e48b045528f5bfe627b6ad554ff32def154f42372786903b7abcfe1aa16"},
|
||||
]
|
||||
|
||||
[package.extras]
|
||||
test = ["pytest (>=6)"]
|
||||
|
||||
[[package]]
|
||||
name = "idna"
|
||||
version = "3.7"
|
||||
|
@ -146,6 +160,65 @@ files = [
|
|||
{file = "idna-3.7.tar.gz", hash = "sha256:028ff3aadf0609c1fd278d8ea3089299412a7a8b9bd005dd08b9f8285bcb5cfc"},
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "iniconfig"
|
||||
version = "2.0.0"
|
||||
description = "brain-dead simple config-ini parsing"
|
||||
optional = false
|
||||
python-versions = ">=3.7"
|
||||
files = [
|
||||
{file = "iniconfig-2.0.0-py3-none-any.whl", hash = "sha256:b6a85871a79d2e3b22d2d1b94ac2824226a63c6b741c88f7ae975f18b6778374"},
|
||||
{file = "iniconfig-2.0.0.tar.gz", hash = "sha256:2d91e135bf72d31a410b17c16da610a82cb55f6b0477d1a902134b24a455b8b3"},
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "packaging"
|
||||
version = "24.0"
|
||||
description = "Core utilities for Python packages"
|
||||
optional = false
|
||||
python-versions = ">=3.7"
|
||||
files = [
|
||||
{file = "packaging-24.0-py3-none-any.whl", hash = "sha256:2ddfb553fdf02fb784c234c7ba6ccc288296ceabec964ad2eae3777778130bc5"},
|
||||
{file = "packaging-24.0.tar.gz", hash = "sha256:eb82c5e3e56209074766e6885bb04b8c38a0c015d0a30036ebe7ece34c9989e9"},
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "pluggy"
|
||||
version = "1.4.0"
|
||||
description = "plugin and hook calling mechanisms for python"
|
||||
optional = false
|
||||
python-versions = ">=3.8"
|
||||
files = [
|
||||
{file = "pluggy-1.4.0-py3-none-any.whl", hash = "sha256:7db9f7b503d67d1c5b95f59773ebb58a8c1c288129a88665838012cfb07b8981"},
|
||||
{file = "pluggy-1.4.0.tar.gz", hash = "sha256:8c85c2876142a764e5b7548e7d9a0e0ddb46f5185161049a79b7e974454223be"},
|
||||
]
|
||||
|
||||
[package.extras]
|
||||
dev = ["pre-commit", "tox"]
|
||||
testing = ["pytest", "pytest-benchmark"]
|
||||
|
||||
[[package]]
|
||||
name = "pytest"
|
||||
version = "8.1.1"
|
||||
description = "pytest: simple powerful testing with Python"
|
||||
optional = false
|
||||
python-versions = ">=3.8"
|
||||
files = [
|
||||
{file = "pytest-8.1.1-py3-none-any.whl", hash = "sha256:2a8386cfc11fa9d2c50ee7b2a57e7d898ef90470a7a34c4b949ff59662bb78b7"},
|
||||
{file = "pytest-8.1.1.tar.gz", hash = "sha256:ac978141a75948948817d360297b7aae0fcb9d6ff6bc9ec6d514b85d5a65c044"},
|
||||
]
|
||||
|
||||
[package.dependencies]
|
||||
colorama = {version = "*", markers = "sys_platform == \"win32\""}
|
||||
exceptiongroup = {version = ">=1.0.0rc8", markers = "python_version < \"3.11\""}
|
||||
iniconfig = "*"
|
||||
packaging = "*"
|
||||
pluggy = ">=1.4,<2.0"
|
||||
tomli = {version = ">=1", markers = "python_version < \"3.11\""}
|
||||
|
||||
[package.extras]
|
||||
testing = ["argcomplete", "attrs (>=19.2)", "hypothesis (>=3.56)", "mock", "pygments (>=2.7.2)", "requests", "setuptools", "xmlschema"]
|
||||
|
||||
[[package]]
|
||||
name = "requests"
|
||||
version = "2.31.0"
|
||||
|
@ -167,6 +240,17 @@ urllib3 = ">=1.21.1,<3"
|
|||
socks = ["PySocks (>=1.5.6,!=1.5.7)"]
|
||||
use-chardet-on-py3 = ["chardet (>=3.0.2,<6)"]
|
||||
|
||||
[[package]]
|
||||
name = "tomli"
|
||||
version = "2.0.1"
|
||||
description = "A lil' TOML parser"
|
||||
optional = false
|
||||
python-versions = ">=3.7"
|
||||
files = [
|
||||
{file = "tomli-2.0.1-py3-none-any.whl", hash = "sha256:939de3e7a6161af0c887ef91b7d41a53e7c5a1ca976325f429cb46ea9bc30ecc"},
|
||||
{file = "tomli-2.0.1.tar.gz", hash = "sha256:de526c12914f0c550d15924c62d72abc48d6fe7364aa87328337a31007fe8a4f"},
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "urllib3"
|
||||
version = "2.2.1"
|
||||
|
@ -187,4 +271,4 @@ zstd = ["zstandard (>=0.18.0)"]
|
|||
[metadata]
|
||||
lock-version = "2.0"
|
||||
python-versions = "^3.9"
|
||||
content-hash = "77fb072556c86c1be54da4a6fcada5234ce37943166ee26a3fa19907fedfc8c6"
|
||||
content-hash = "3be6f3b93ec28b49943b98f17c6854993da146534477f47044346a461d868e31"
|
||||
|
|
|
@ -25,6 +25,9 @@ click = "^8.1.7"
|
|||
requests = "^2.31.0"
|
||||
|
||||
|
||||
[tool.poetry.group.dev.dependencies]
|
||||
pytest = "^8.1.1"
|
||||
|
||||
[build-system]
|
||||
requires = ["poetry-core"]
|
||||
build-backend = "poetry.core.masonry.api"
|
||||
|
|
0
tests/__init__.py
Normal file
0
tests/__init__.py
Normal file
115
tests/test_config.py
Normal file
115
tests/test_config.py
Normal file
|
@ -0,0 +1,115 @@
|
|||
from datetime import datetime
|
||||
|
||||
from cosmicweb_music.cosmICweb import (
|
||||
apply_config_parameter,
|
||||
music_config_to_template,
|
||||
process_config,
|
||||
DEFAULT_URL,
|
||||
compose_template,
|
||||
)
|
||||
from cosmicweb_music.data_types import DownloadConfig, Args, Ellipsoid
|
||||
|
||||
some_config = """
|
||||
[setup]
|
||||
boxlength = 150
|
||||
zstart = 99
|
||||
levelmin = 8
|
||||
levelmin_TF = 8
|
||||
levelmax = 12
|
||||
padding = 16 # try reduce it at your own risk
|
||||
overlap = 4
|
||||
overlap = 4
|
||||
align_top = no
|
||||
baryons = no # switch on for baryon runs
|
||||
use_2LPT = no
|
||||
use_LLA = no # AMR codes might want to enable this
|
||||
""".strip()
|
||||
|
||||
some_config_modified = """
|
||||
[setup]
|
||||
boxlength = 150
|
||||
zstart = 123
|
||||
levelmin = 8
|
||||
levelmin_TF = 8
|
||||
levelmax = 12
|
||||
padding = 16 # try reduce it at your own risk
|
||||
overlap = 4
|
||||
overlap = 4
|
||||
align_top = no
|
||||
baryons = no # switch on for baryon runs
|
||||
use_2LPT = no
|
||||
use_LLA = no # AMR codes might want to enable this
|
||||
""".strip()
|
||||
|
||||
config = DownloadConfig(
|
||||
simulation_name="test",
|
||||
project_name="project",
|
||||
halo_names=["halo_1"],
|
||||
halo_ids=[1],
|
||||
halo_urls=["..."],
|
||||
traceback_radius=10,
|
||||
api_token="...",
|
||||
MUSIC={"cosmology": "", "poisson": "", "random": "", "setup": ""},
|
||||
settings=None,
|
||||
accessed_at=datetime.now(),
|
||||
)
|
||||
|
||||
args = Args(DEFAULT_URL, ".", False, 1)
|
||||
ellipsoid = Ellipsoid(
|
||||
radius_definition="Rvir",
|
||||
center=[0.42174551551333334, 0.42890526632, 0.27975938776000003],
|
||||
shape=[
|
||||
[872.2886068575001, -31.76815629375, 92.22811563824999],
|
||||
[-31.76815629375, 520.6134379275, 28.34206946775],
|
||||
[92.22811563824999, 28.34206946775, 165.70762251300002],
|
||||
],
|
||||
traceback_radius=10.0,
|
||||
)
|
||||
|
||||
generated_config = """
|
||||
# Zoom Initial Conditions for halo 1 (halo_1) in simulation test (project project)
|
||||
# Details on this halo can be found on https://cosmicweb.eu/simulation/test/halo/1
|
||||
# This file has been generated by CosmICweb @2021-01-01T00:00:00
|
||||
|
||||
|
||||
[setup]
|
||||
|
||||
|
||||
# Ellipsoidal refinement region defined on unity cube
|
||||
# This minimum bounding ellipsoid has been obtained from
|
||||
# particles within 10.0 Rvir of the halo center
|
||||
region = ellipsoid
|
||||
region_ellipsoid_matrix[0] = 872.2886068575001, -31.76815629375, 92.22811563824999
|
||||
region_ellipsoid_matrix[1] = -31.76815629375, 520.6134379275, 28.34206946775
|
||||
region_ellipsoid_matrix[2] = 92.22811563824999, 28.34206946775, 165.70762251300002
|
||||
region_ellipsoid_center = 0.42174551551333334, 0.42890526632, 0.27975938776000003
|
||||
|
||||
|
||||
[cosmology]
|
||||
|
||||
|
||||
[random]
|
||||
|
||||
|
||||
[poisson]
|
||||
|
||||
""".lstrip()
|
||||
|
||||
|
||||
def test_apply_config_parameter_empty():
|
||||
assert apply_config_parameter(some_config, {}) == some_config
|
||||
|
||||
|
||||
def test_apply_config_parameter():
|
||||
params = {"zstart": 123, "doesn't exist": 1}
|
||||
assert apply_config_parameter(some_config, params) == some_config_modified
|
||||
|
||||
|
||||
def test_music_config_to_template():
|
||||
halo_name = "halo_1"
|
||||
halo_id = 1
|
||||
music_template = music_config_to_template(config)
|
||||
music_config = compose_template(
|
||||
music_template, ellipsoid, config, halo_name, halo_id, now=datetime(2021, 1, 1)
|
||||
)
|
||||
assert music_config == generated_config
|
146
tests/test_e2e.py
Normal file
146
tests/test_e2e.py
Normal file
|
@ -0,0 +1,146 @@
|
|||
import re
|
||||
from datetime import datetime
|
||||
|
||||
from cosmicweb_music.cosmICweb import (
|
||||
apply_config_parameter,
|
||||
music_config_to_template,
|
||||
process_config,
|
||||
DEFAULT_URL,
|
||||
compose_template,
|
||||
downloadstore_mode,
|
||||
publication_mode,
|
||||
)
|
||||
from cosmicweb_music.data_types import DownloadConfig, Args, Ellipsoid
|
||||
|
||||
reference_output = """
|
||||
# Zoom Initial Conditions for halo 208416759 (halo_208416759) in simulation 150MPC (CosmOCA project)
|
||||
# Details on this halo can be found on https://cosmicweb.eu/simulation/150MPC/halo/208416759
|
||||
# This file has been generated by CosmICweb @2024-04-20T22:26:13.916577
|
||||
|
||||
|
||||
[setup]
|
||||
boxlength = 150
|
||||
zstart = 99
|
||||
levelmin = 8
|
||||
levelmin_TF = 8
|
||||
levelmax = 12
|
||||
padding = 16 # try reduce it at your own risk
|
||||
overlap = 4
|
||||
align_top = no
|
||||
baryons = no # switch on for baryon runs
|
||||
use_2LPT = no
|
||||
use_LLA = no # AMR codes might want to enable this
|
||||
|
||||
# Ellipsoidal refinement region defined on unity cube
|
||||
# This minimum bounding ellipsoid has been obtained from
|
||||
# particles within 10.0 Rvir of the halo center
|
||||
region = ellipsoid
|
||||
region_ellipsoid_matrix[0] = 872.2886068575001, -31.76815629375, 92.22811563824999
|
||||
region_ellipsoid_matrix[1] = -31.76815629375, 520.6134379275, 28.34206946775
|
||||
region_ellipsoid_matrix[2] = 92.22811563824999, 28.34206946775, 165.70762251300002
|
||||
region_ellipsoid_center = 0.42174551551333334, 0.42890526632, 0.27975938776000003
|
||||
|
||||
|
||||
[cosmology]
|
||||
Omega_m = 0.309
|
||||
Omega_L = 0.691
|
||||
Omega_b = 0.049
|
||||
H0 = 67.74
|
||||
sigma_8 = 0.816
|
||||
nspec = 0.9667
|
||||
transfer = eisenstein
|
||||
|
||||
[random]
|
||||
cubesize = 256
|
||||
seed[9] = 74927
|
||||
seed[10] = 21450
|
||||
|
||||
[poisson]
|
||||
fft_fine = true
|
||||
accuracy = 1e-5
|
||||
grad_order = 4
|
||||
laplace_order = 4
|
||||
""".lstrip()
|
||||
|
||||
reference_output_publication="""
|
||||
# Zoom Initial Conditions for halo 25505622 (1e11v) in simulation AGORA (RHAPSODY project)
|
||||
# Details on this halo can be found on https://cosmicweb.eu/simulation/AGORA/halo/25505622
|
||||
# This file has been generated by CosmICweb @2024-04-21T00:03:24.827918
|
||||
|
||||
|
||||
[setup]
|
||||
boxlength = 60
|
||||
zstart = 100
|
||||
levelmin = 9
|
||||
levelmin_TF = 9
|
||||
levelmax = 9
|
||||
padding = 16 # try reduce it at your own risk
|
||||
overlap = 4
|
||||
align_top = no
|
||||
baryons = no # switch on for baryon runs
|
||||
use_2LPT = no
|
||||
use_LLA = no # AMR codes might want to enable this
|
||||
|
||||
# Ellipsoidal refinement region defined on unity cube
|
||||
# This minimum bounding ellipsoid has been obtained from
|
||||
# particles within 2.0 Rvir of the halo center
|
||||
region = ellipsoid
|
||||
region_ellipsoid_matrix[0] = 1202.685817644, -224.73030332520003, 78.4954201104
|
||||
region_ellipsoid_matrix[1] = -224.73030332520003, 1126.675415484, -514.163771388
|
||||
region_ellipsoid_matrix[2] = 78.4954201104, -514.163771388, 859.827522564
|
||||
region_ellipsoid_center = 0.6253459789833333, 0.47749109738333334, 0.6903304682000001
|
||||
|
||||
|
||||
[cosmology]
|
||||
Omega_m = 0.272
|
||||
Omega_L = 0.728
|
||||
Omega_b = 0.0455
|
||||
H0 = 70.2
|
||||
sigma_8 = 0.807
|
||||
nspec = 0.961
|
||||
transfer = eisenstein
|
||||
#below are MUSIC defaults to initialize gas temperature for some codes
|
||||
#YHe = 0.248 # primordial He abundance
|
||||
#gamma = 1.6667 # adiabatic exponent (=5/3)
|
||||
|
||||
[random]
|
||||
cubesize = 256
|
||||
seed[8] = 95064
|
||||
seed[9] = 31415
|
||||
seed[10] = 27183
|
||||
# do not add higher seeds!
|
||||
|
||||
[poisson]
|
||||
fft_fine = yes
|
||||
accuracy = 1e-6
|
||||
grad_order = 6
|
||||
laplace_order = 6
|
||||
""".lstrip()
|
||||
|
||||
time_fix_regex = re.compile(r"@[\d\-T:.]+")
|
||||
|
||||
|
||||
def test_single_saved():
|
||||
id = "f5399734-ad67-432b-ba4d-61bc2088136a"
|
||||
args = Args(output_path="./", url=DEFAULT_URL, common_directory=True, attempts=1)
|
||||
output = downloadstore_mode(args, id, store=False)
|
||||
assert len(output) == 1
|
||||
output = output[0]
|
||||
assert output[0] == "./cosmICweb-zooms-150MPC/ics_halo_208416759.cfg"
|
||||
|
||||
assert time_fix_regex.sub("TIME", output[1]) == time_fix_regex.sub(
|
||||
"TIME", reference_output
|
||||
)
|
||||
|
||||
|
||||
def test_publication():
|
||||
id = "agora-halos"
|
||||
args = Args(output_path="./", url="http://127.0.0.1:5000", common_directory=True, attempts=1)
|
||||
output = publication_mode(args, id, store=False, traceback_radius=2.0)
|
||||
assert len(output) == 6
|
||||
output = output[0]
|
||||
assert output[0] == "./agora-halos/1e11v/ics.cfg"
|
||||
|
||||
assert time_fix_regex.sub("TIME", output[1]) == time_fix_regex.sub(
|
||||
"TIME", reference_output_publication
|
||||
)
|
Loading…
Reference in a new issue