mirror of
https://github.com/cosmo-sims/cosmICweb-music.git
synced 2024-09-19 16:53:43 +02:00
allow downloading public collections
This commit is contained in:
parent
f00726063d
commit
68b9dd5465
3 changed files with 124 additions and 8 deletions
|
@ -24,6 +24,7 @@ Options:
|
||||||
--help Show this message and exit.
|
--help Show this message and exit.
|
||||||
|
|
||||||
Commands:
|
Commands:
|
||||||
|
collection Download shared ICs using the collection UUID
|
||||||
get Download ICs using a target UUID generated on cosmICweb
|
get Download ICs using a target UUID generated on cosmICweb
|
||||||
publication Download published ICs using the publication name
|
publication Download published ICs using the publication name
|
||||||
```
|
```
|
||||||
|
@ -34,3 +35,6 @@ cosmicweb-music publication agora-halos
|
||||||
```bash
|
```bash
|
||||||
cosmicweb-music get f5399734-ad67-432b-ba4d-61bc2088136a
|
cosmicweb-music get f5399734-ad67-432b-ba4d-61bc2088136a
|
||||||
```
|
```
|
||||||
|
```bash
|
||||||
|
cosmicweb-music collection c30de0f3-ab4d-48ad-aa26-f20bb4b70bbd
|
||||||
|
```
|
||||||
|
|
|
@ -126,11 +126,20 @@ def fetch_downloadstore(cosmicweb_url: str, target: str) -> DownloadConfig:
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
def fetch_publication(
|
def fetch_multiple(
|
||||||
cosmicweb_url: str, publication_name: str, traceback_radius
|
cosmicweb_url: str,
|
||||||
|
traceback_radius,
|
||||||
|
publication_name: str = None,
|
||||||
|
collection_uuid: str = None,
|
||||||
) -> DownloadConfig:
|
) -> DownloadConfig:
|
||||||
|
if publication_name:
|
||||||
|
url = f"{cosmicweb_url}/api/publications/{publication_name}"
|
||||||
|
elif collection_uuid:
|
||||||
|
url = f"{cosmicweb_url}/api/collections/{collection_uuid}"
|
||||||
|
else:
|
||||||
|
raise ValueError("must provide either publication_name or collection_uuid")
|
||||||
try:
|
try:
|
||||||
r = requests.get(cosmicweb_url + "/api/publications/" + publication_name)
|
r = requests.get(url)
|
||||||
# This will raise an error if not successful
|
# This will raise an error if not successful
|
||||||
r.raise_for_status()
|
r.raise_for_status()
|
||||||
except requests.exceptions.HTTPError as e:
|
except requests.exceptions.HTTPError as e:
|
||||||
|
@ -139,7 +148,12 @@ def fetch_publication(
|
||||||
sys.exit(1)
|
sys.exit(1)
|
||||||
content = r.json()
|
content = r.json()
|
||||||
sim = content["simulation"]
|
sim = content["simulation"]
|
||||||
halo_names = [h["name"] for h in content["halos"]]
|
halo_names = []
|
||||||
|
for h in content["halos"]:
|
||||||
|
name = h["name"]
|
||||||
|
if name is None:
|
||||||
|
name = str(h["id"])
|
||||||
|
halo_names.append(name)
|
||||||
halo_ids = [h["id"] for h in content["halos"]]
|
halo_ids = [h["id"] for h in content["halos"]]
|
||||||
halo_urls = [
|
halo_urls = [
|
||||||
"{url}/simulation/{sid}/halo/{hid}".format(
|
"{url}/simulation/{sid}/halo/{hid}".format(
|
||||||
|
@ -185,8 +199,14 @@ def apply_config_parameter(config: str, parameters: dict[str, Any]) -> str:
|
||||||
return "\n".join(new_lines)
|
return "\n".join(new_lines)
|
||||||
|
|
||||||
|
|
||||||
|
def normalize_lineendings(text: str) -> str:
|
||||||
|
return text.replace("\r\n", "\n")
|
||||||
|
|
||||||
|
|
||||||
def music_config_to_template(config: DownloadConfig) -> str:
|
def music_config_to_template(config: DownloadConfig) -> str:
|
||||||
music_config = config.MUSIC
|
music_config = config.MUSIC
|
||||||
|
music_config = {k: normalize_lineendings(v) for k, v in music_config.items()}
|
||||||
|
|
||||||
settings = config.settings
|
settings = config.settings
|
||||||
# TODO: apply output configuration
|
# TODO: apply output configuration
|
||||||
config = (
|
config = (
|
||||||
|
@ -336,16 +356,30 @@ def downloadstore_mode(args: Args, target: str, store=True) -> None | str:
|
||||||
def publication_mode(
|
def publication_mode(
|
||||||
args: Args, publication_name: str, traceback_radius, store=True
|
args: Args, publication_name: str, traceback_radius, store=True
|
||||||
) -> None | str:
|
) -> None | str:
|
||||||
logging.info(
|
logging.info(f"Fetching publication {publication_name} from the cosmICweb server")
|
||||||
"Fetching publication " + publication_name + " from the cosmICweb server"
|
config = fetch_multiple(
|
||||||
|
args.url, traceback_radius, publication_name=publication_name
|
||||||
)
|
)
|
||||||
config = fetch_publication(args.url, publication_name, traceback_radius)
|
|
||||||
args = args._replace(output_path=os.path.join(args.output_path, publication_name))
|
args = args._replace(output_path=os.path.join(args.output_path, publication_name))
|
||||||
logging.debug("Output directory set to " + args.output_path)
|
logging.debug("Output directory set to " + args.output_path)
|
||||||
logging.info("Publication successfully fetched")
|
logging.info("Publication successfully fetched")
|
||||||
return process_config(config, args, store)
|
return process_config(config, args, store)
|
||||||
|
|
||||||
|
|
||||||
|
def collection_mode(
|
||||||
|
args: Args, collection_uuid: str, traceback_radius, store=True
|
||||||
|
) -> None | str:
|
||||||
|
logging.info(f"Fetching collection {collection_uuid} from the cosmICweb server")
|
||||||
|
config = fetch_multiple(args.url, traceback_radius, collection_uuid=collection_uuid)
|
||||||
|
args = args._replace(
|
||||||
|
output_path=os.path.join(args.output_path, config.simulation_name)
|
||||||
|
)
|
||||||
|
logging.debug("Output directory set to " + args.output_path)
|
||||||
|
logging.info("Publication successfully fetched")
|
||||||
|
print(config)
|
||||||
|
return process_config(config, args, store)
|
||||||
|
|
||||||
|
|
||||||
def dir_path(p: str) -> str:
|
def dir_path(p: str) -> str:
|
||||||
if os.path.isdir(p):
|
if os.path.isdir(p):
|
||||||
return p
|
return p
|
||||||
|
@ -404,5 +438,17 @@ def publication(ctx, publication_name, traceback_radius):
|
||||||
publication_mode(args, publication_name, traceback_radius)
|
publication_mode(args, publication_name, traceback_radius)
|
||||||
|
|
||||||
|
|
||||||
|
@cli.command(help="Download shared ICs using the collection UUID")
|
||||||
|
@click.argument("collection")
|
||||||
|
@click.option(
|
||||||
|
"--traceback_radius", type=click.Choice(["1", "2", "4", "10"]), default="2"
|
||||||
|
)
|
||||||
|
@click.pass_context
|
||||||
|
def collection(ctx, collection, traceback_radius):
|
||||||
|
traceback_radius = float(traceback_radius)
|
||||||
|
args: Args = ctx.obj
|
||||||
|
collection_mode(args, collection, traceback_radius)
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
if __name__ == "__main__":
|
||||||
cli()
|
cli()
|
||||||
|
|
|
@ -4,6 +4,7 @@ from cosmicweb_music.cosmICweb import (
|
||||||
DEFAULT_URL,
|
DEFAULT_URL,
|
||||||
downloadstore_mode,
|
downloadstore_mode,
|
||||||
publication_mode,
|
publication_mode,
|
||||||
|
collection_mode,
|
||||||
)
|
)
|
||||||
from cosmicweb_music.data_types import Args
|
from cosmicweb_music.data_types import Args
|
||||||
|
|
||||||
|
@ -70,7 +71,7 @@ ramses_pvar_idx = 43
|
||||||
|
|
||||||
""".lstrip()
|
""".lstrip()
|
||||||
|
|
||||||
reference_output_publication="""
|
reference_output_publication = """
|
||||||
# Zoom Initial Conditions for halo 25505622 (1e11v) in simulation AGORA (AGORA Project project)
|
# Zoom Initial Conditions for halo 25505622 (1e11v) in simulation AGORA (AGORA Project project)
|
||||||
# Details on this halo can be found on https://cosmicweb.eu/simulation/AGORA/halo/25505622
|
# Details on this halo can be found on https://cosmicweb.eu/simulation/AGORA/halo/25505622
|
||||||
# This file has been generated by CosmICweb @2024-04-21T00:03:24.827918
|
# This file has been generated by CosmICweb @2024-04-21T00:03:24.827918
|
||||||
|
@ -128,6 +129,58 @@ laplace_order = 6
|
||||||
# TODO: add output options
|
# TODO: add output options
|
||||||
""".lstrip()
|
""".lstrip()
|
||||||
|
|
||||||
|
reference_output_collection="""
|
||||||
|
# Zoom Initial Conditions for halo 20770177 (20770177) in simulation RefL0100N1504 (EAGLE Simulations project)
|
||||||
|
# Details on this halo can be found on https://cosmicweb.eu/simulation/RefL0100N1504/halo/20770177
|
||||||
|
# This file has been generated by CosmICweb @2024-04-27T19:00:39.574652
|
||||||
|
|
||||||
|
|
||||||
|
[setup]
|
||||||
|
boxlength = 67.77
|
||||||
|
zstart = 127
|
||||||
|
levelmin = 7
|
||||||
|
levelmin_TF = 9
|
||||||
|
levelmax = 11
|
||||||
|
baryons = no
|
||||||
|
use_2LPT = no
|
||||||
|
use_LLA = no
|
||||||
|
padding = 8
|
||||||
|
|
||||||
|
# Ellipsoidal refinement region defined on unity cube
|
||||||
|
# This minimum bounding ellipsoid has been obtained from
|
||||||
|
# particles within 2.0 R200c of the halo center
|
||||||
|
region = ellipsoid
|
||||||
|
region_ellipsoid_matrix[0] = 180.08392, 30.495844, 5.7450113
|
||||||
|
region_ellipsoid_matrix[1] = 30.495844, 247.56755, 45.540962
|
||||||
|
region_ellipsoid_matrix[2] = 5.7450113, 45.540962, 377.32254
|
||||||
|
region_ellipsoid_center = 0.92963797, 0.3582335, 0.9368965
|
||||||
|
|
||||||
|
|
||||||
|
[cosmology]
|
||||||
|
Omega_m = 0.307
|
||||||
|
Omega_L = 0.693
|
||||||
|
Omega_b = 0.04825
|
||||||
|
H0 = 67.77
|
||||||
|
sigma_8 = 0.8288
|
||||||
|
nspec = 0.9611
|
||||||
|
ZeroRadiation = true
|
||||||
|
transfer = camb_file
|
||||||
|
transfer_file = Eagle_camb_file.txt
|
||||||
|
|
||||||
|
[random]
|
||||||
|
generator = PANPHASIA
|
||||||
|
descriptor =[Panph1,L16,(31250,23438,39063),S12,CH1050187043,EAGLE_L0100_VOL1]
|
||||||
|
|
||||||
|
[poisson]
|
||||||
|
fft_fine = yes
|
||||||
|
accuracy = 1e-6
|
||||||
|
grad_order = 6
|
||||||
|
laplace_order = 6
|
||||||
|
|
||||||
|
[output]
|
||||||
|
# TODO: add output options
|
||||||
|
""".lstrip()
|
||||||
|
|
||||||
time_fix_regex = re.compile(r"@[\d\-T:.]+")
|
time_fix_regex = re.compile(r"@[\d\-T:.]+")
|
||||||
|
|
||||||
|
|
||||||
|
@ -155,3 +208,16 @@ def test_publication():
|
||||||
assert time_fix_regex.sub("TIME", output[1]) == time_fix_regex.sub(
|
assert time_fix_regex.sub("TIME", output[1]) == time_fix_regex.sub(
|
||||||
"TIME", reference_output_publication
|
"TIME", reference_output_publication
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def test_collection():
|
||||||
|
id = "c30de0f3-ab4d-48ad-aa26-f20bb4b70bbd"
|
||||||
|
args = Args(output_path="./", url=DEFAULT_URL, common_directory=True, attempts=1)
|
||||||
|
output = collection_mode(args, id, store=False, traceback_radius=2.0)
|
||||||
|
assert len(output) == 1
|
||||||
|
output = output[0]
|
||||||
|
assert output[0] == "./RefL0100N1504/ics_20770177.cfg"
|
||||||
|
|
||||||
|
assert time_fix_regex.sub("TIME", output[1]) == time_fix_regex.sub(
|
||||||
|
"TIME", reference_output_collection
|
||||||
|
)
|
||||||
|
|
Loading…
Reference in a new issue