From 68b9dd5465f33271f779fb0d504eabbe8e8c33be Mon Sep 17 00:00:00 2001 From: Lukas Winkler Date: Sat, 27 Apr 2024 19:11:16 +0200 Subject: [PATCH] allow downloading public collections --- README.md | 4 +++ cosmicweb_music/cosmICweb.py | 60 +++++++++++++++++++++++++++---- tests/test_e2e.py | 68 +++++++++++++++++++++++++++++++++++- 3 files changed, 124 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index 97628da..eda819a 100644 --- a/README.md +++ b/README.md @@ -24,6 +24,7 @@ Options: --help Show this message and exit. Commands: + collection Download shared ICs using the collection UUID get Download ICs using a target UUID generated on cosmICweb publication Download published ICs using the publication name ``` @@ -34,3 +35,6 @@ cosmicweb-music publication agora-halos ```bash cosmicweb-music get f5399734-ad67-432b-ba4d-61bc2088136a ``` +```bash +cosmicweb-music collection c30de0f3-ab4d-48ad-aa26-f20bb4b70bbd +``` diff --git a/cosmicweb_music/cosmICweb.py b/cosmicweb_music/cosmICweb.py index 5efcbb2..e22b760 100755 --- a/cosmicweb_music/cosmICweb.py +++ b/cosmicweb_music/cosmICweb.py @@ -126,11 +126,20 @@ def fetch_downloadstore(cosmicweb_url: str, target: str) -> DownloadConfig: ) -def fetch_publication( - cosmicweb_url: str, publication_name: str, traceback_radius +def fetch_multiple( + cosmicweb_url: str, + traceback_radius, + publication_name: str = None, + collection_uuid: str = None, ) -> DownloadConfig: + if publication_name: + url = f"{cosmicweb_url}/api/publications/{publication_name}" + elif collection_uuid: + url = f"{cosmicweb_url}/api/collections/{collection_uuid}" + else: + raise ValueError("must provide either publication_name or collection_uuid") try: - r = requests.get(cosmicweb_url + "/api/publications/" + publication_name) + r = requests.get(url) # This will raise an error if not successful r.raise_for_status() except requests.exceptions.HTTPError as e: @@ -139,7 +148,12 @@ def fetch_publication( sys.exit(1) content = r.json() sim = content["simulation"] - halo_names = [h["name"] for h in content["halos"]] + halo_names = [] + for h in content["halos"]: + name = h["name"] + if name is None: + name = str(h["id"]) + halo_names.append(name) halo_ids = [h["id"] for h in content["halos"]] halo_urls = [ "{url}/simulation/{sid}/halo/{hid}".format( @@ -185,8 +199,14 @@ def apply_config_parameter(config: str, parameters: dict[str, Any]) -> str: return "\n".join(new_lines) +def normalize_lineendings(text: str) -> str: + return text.replace("\r\n", "\n") + + def music_config_to_template(config: DownloadConfig) -> str: music_config = config.MUSIC + music_config = {k: normalize_lineendings(v) for k, v in music_config.items()} + settings = config.settings # TODO: apply output configuration config = ( @@ -336,16 +356,30 @@ def downloadstore_mode(args: Args, target: str, store=True) -> None | str: def publication_mode( args: Args, publication_name: str, traceback_radius, store=True ) -> None | str: - logging.info( - "Fetching publication " + publication_name + " from the cosmICweb server" + logging.info(f"Fetching publication {publication_name} from the cosmICweb server") + config = fetch_multiple( + args.url, traceback_radius, publication_name=publication_name ) - config = fetch_publication(args.url, publication_name, traceback_radius) args = args._replace(output_path=os.path.join(args.output_path, publication_name)) logging.debug("Output directory set to " + args.output_path) logging.info("Publication successfully fetched") return process_config(config, args, store) +def collection_mode( + args: Args, collection_uuid: str, traceback_radius, store=True +) -> None | str: + logging.info(f"Fetching collection {collection_uuid} from the cosmICweb server") + config = fetch_multiple(args.url, traceback_radius, collection_uuid=collection_uuid) + args = args._replace( + output_path=os.path.join(args.output_path, config.simulation_name) + ) + logging.debug("Output directory set to " + args.output_path) + logging.info("Publication successfully fetched") + print(config) + return process_config(config, args, store) + + def dir_path(p: str) -> str: if os.path.isdir(p): return p @@ -404,5 +438,17 @@ def publication(ctx, publication_name, traceback_radius): publication_mode(args, publication_name, traceback_radius) +@cli.command(help="Download shared ICs using the collection UUID") +@click.argument("collection") +@click.option( + "--traceback_radius", type=click.Choice(["1", "2", "4", "10"]), default="2" +) +@click.pass_context +def collection(ctx, collection, traceback_radius): + traceback_radius = float(traceback_radius) + args: Args = ctx.obj + collection_mode(args, collection, traceback_radius) + + if __name__ == "__main__": cli() diff --git a/tests/test_e2e.py b/tests/test_e2e.py index dd2ba5c..cb16623 100644 --- a/tests/test_e2e.py +++ b/tests/test_e2e.py @@ -4,6 +4,7 @@ from cosmicweb_music.cosmICweb import ( DEFAULT_URL, downloadstore_mode, publication_mode, + collection_mode, ) from cosmicweb_music.data_types import Args @@ -70,7 +71,7 @@ ramses_pvar_idx = 43 """.lstrip() -reference_output_publication=""" +reference_output_publication = """ # Zoom Initial Conditions for halo 25505622 (1e11v) in simulation AGORA (AGORA Project project) # Details on this halo can be found on https://cosmicweb.eu/simulation/AGORA/halo/25505622 # This file has been generated by CosmICweb @2024-04-21T00:03:24.827918 @@ -128,6 +129,58 @@ laplace_order = 6 # TODO: add output options """.lstrip() +reference_output_collection=""" +# Zoom Initial Conditions for halo 20770177 (20770177) in simulation RefL0100N1504 (EAGLE Simulations project) +# Details on this halo can be found on https://cosmicweb.eu/simulation/RefL0100N1504/halo/20770177 +# This file has been generated by CosmICweb @2024-04-27T19:00:39.574652 + + +[setup] +boxlength = 67.77 +zstart = 127 +levelmin = 7 +levelmin_TF = 9 +levelmax = 11 +baryons = no +use_2LPT = no +use_LLA = no +padding = 8 + +# Ellipsoidal refinement region defined on unity cube +# This minimum bounding ellipsoid has been obtained from +# particles within 2.0 R200c of the halo center +region = ellipsoid +region_ellipsoid_matrix[0] = 180.08392, 30.495844, 5.7450113 +region_ellipsoid_matrix[1] = 30.495844, 247.56755, 45.540962 +region_ellipsoid_matrix[2] = 5.7450113, 45.540962, 377.32254 +region_ellipsoid_center = 0.92963797, 0.3582335, 0.9368965 + + +[cosmology] +Omega_m = 0.307 +Omega_L = 0.693 +Omega_b = 0.04825 +H0 = 67.77 +sigma_8 = 0.8288 +nspec = 0.9611 +ZeroRadiation = true +transfer = camb_file +transfer_file = Eagle_camb_file.txt + +[random] +generator = PANPHASIA +descriptor =[Panph1,L16,(31250,23438,39063),S12,CH1050187043,EAGLE_L0100_VOL1] + +[poisson] +fft_fine = yes +accuracy = 1e-6 +grad_order = 6 +laplace_order = 6 + +[output] +# TODO: add output options +""".lstrip() + time_fix_regex = re.compile(r"@[\d\-T:.]+") @@ -155,3 +208,16 @@ def test_publication(): assert time_fix_regex.sub("TIME", output[1]) == time_fix_regex.sub( "TIME", reference_output_publication ) + + +def test_collection(): + id = "c30de0f3-ab4d-48ad-aa26-f20bb4b70bbd" + args = Args(output_path="./", url=DEFAULT_URL, common_directory=True, attempts=1) + output = collection_mode(args, id, store=False, traceback_radius=2.0) + assert len(output) == 1 + output = output[0] + assert output[0] == "./RefL0100N1504/ics_20770177.cfg" + + assert time_fix_regex.sub("TIME", output[1]) == time_fix_regex.sub( + "TIME", reference_output_collection + )