allow downloading public collections

README.md

@@ -24,6 +24,7 @@ Options:
   --help                  Show this message and exit.
 
 Commands:
+  collection   Download shared ICs using the collection UUID
   get          Download ICs using a target UUID generated on cosmICweb
   publication  Download published ICs using the publication name
 ```
@@ -34,3 +35,6 @@ cosmicweb-music publication agora-halos
 ```bash
 cosmicweb-music get f5399734-ad67-432b-ba4d-61bc2088136a
 ```
+```bash
+cosmicweb-music collection c30de0f3-ab4d-48ad-aa26-f20bb4b70bbd
+```

cosmicweb_music/cosmICweb.py

@@ -126,11 +126,20 @@ def fetch_downloadstore(cosmicweb_url: str, target: str) -> DownloadConfig:
     )
 
 
-def fetch_publication(
-    cosmicweb_url: str, publication_name: str, traceback_radius
+def fetch_multiple(
+    cosmicweb_url: str,
+    traceback_radius,
+    publication_name: str = None,
+    collection_uuid: str = None,
 ) -> DownloadConfig:
+    if publication_name:
+        url = f"{cosmicweb_url}/api/publications/{publication_name}"
+    elif collection_uuid:
+        url = f"{cosmicweb_url}/api/collections/{collection_uuid}"
+    else:
+        raise ValueError("must provide either publication_name or collection_uuid")
     try:
-        r = requests.get(cosmicweb_url + "/api/publications/" + publication_name)
+        r = requests.get(url)
         # This will raise an error if not successful
         r.raise_for_status()
     except requests.exceptions.HTTPError as e:
@@ -139,7 +148,12 @@ def fetch_publication(
         sys.exit(1)
     content = r.json()
     sim = content["simulation"]
-    halo_names = [h["name"] for h in content["halos"]]
+    halo_names = []
+    for h in content["halos"]:
+        name = h["name"]
+        if name is None:
+            name = str(h["id"])
+        halo_names.append(name)
     halo_ids = [h["id"] for h in content["halos"]]
     halo_urls = [
         "{url}/simulation/{sid}/halo/{hid}".format(
@@ -185,8 +199,14 @@ def apply_config_parameter(config: str, parameters: dict[str, Any]) -> str:
     return "\n".join(new_lines)
 
 
+def normalize_lineendings(text: str) -> str:
+    return text.replace("\r\n", "\n")
+
+
 def music_config_to_template(config: DownloadConfig) -> str:
     music_config = config.MUSIC
+    music_config = {k: normalize_lineendings(v) for k, v in music_config.items()}
+
     settings = config.settings
     # TODO: apply output configuration
     config = (
@@ -336,16 +356,30 @@ def downloadstore_mode(args: Args, target: str, store=True) -> None | str:
 def publication_mode(
     args: Args, publication_name: str, traceback_radius, store=True
 ) -> None | str:
-    logging.info(
-        "Fetching publication " + publication_name + " from the cosmICweb server"
+    logging.info(f"Fetching publication {publication_name} from the cosmICweb server")
+    config = fetch_multiple(
+        args.url, traceback_radius, publication_name=publication_name
     )
-    config = fetch_publication(args.url, publication_name, traceback_radius)
     args = args._replace(output_path=os.path.join(args.output_path, publication_name))
     logging.debug("Output directory set to " + args.output_path)
     logging.info("Publication successfully fetched")
     return process_config(config, args, store)
 
 
+def collection_mode(
+    args: Args, collection_uuid: str, traceback_radius, store=True
+) -> None | str:
+    logging.info(f"Fetching collection {collection_uuid} from the cosmICweb server")
+    config = fetch_multiple(args.url, traceback_radius, collection_uuid=collection_uuid)
+    args = args._replace(
+        output_path=os.path.join(args.output_path, config.simulation_name)
+    )
+    logging.debug("Output directory set to " + args.output_path)
+    logging.info("Publication successfully fetched")
+    print(config)
+    return process_config(config, args, store)
+
+
 def dir_path(p: str) -> str:
     if os.path.isdir(p):
         return p
@@ -404,5 +438,17 @@ def publication(ctx, publication_name, traceback_radius):
     publication_mode(args, publication_name, traceback_radius)
 
 
+@cli.command(help="Download shared ICs using the collection UUID")
+@click.argument("collection")
+@click.option(
+    "--traceback_radius", type=click.Choice(["1", "2", "4", "10"]), default="2"
+)
+@click.pass_context
+def collection(ctx, collection, traceback_radius):
+    traceback_radius = float(traceback_radius)
+    args: Args = ctx.obj
+    collection_mode(args, collection, traceback_radius)
+
+
 if __name__ == "__main__":
     cli()

tests/test_e2e.py

@@ -4,6 +4,7 @@ from cosmicweb_music.cosmICweb import (
     DEFAULT_URL,
     downloadstore_mode,
     publication_mode,
+    collection_mode,
 )
 from cosmicweb_music.data_types import Args
 
@@ -70,7 +71,7 @@ ramses_pvar_idx = 43
 
 """.lstrip()
 
-reference_output_publication="""
+reference_output_publication = """
 # Zoom Initial Conditions for halo 25505622 (1e11v) in simulation AGORA (AGORA Project project)
 # Details on this halo can be found on https://cosmicweb.eu/simulation/AGORA/halo/25505622
 # This file has been generated by CosmICweb @2024-04-21T00:03:24.827918
@@ -128,6 +129,58 @@ laplace_order   	= 6
 # TODO: add output options
 """.lstrip()
 
+reference_output_collection="""
+# Zoom Initial Conditions for halo 20770177 (20770177) in simulation RefL0100N1504 (EAGLE Simulations project)
+# Details on this halo can be found on https://cosmicweb.eu/simulation/RefL0100N1504/halo/20770177
+# This file has been generated by CosmICweb @2024-04-27T19:00:39.574652
+
+
+[setup]
+boxlength   = 67.77
+zstart       = 127
+levelmin     = 7
+levelmin_TF  = 9
+levelmax     = 11
+baryons     = no
+use_2LPT    = no
+use_LLA     = no
+padding    = 8
+
+# Ellipsoidal refinement region defined on unity cube
+# This minimum bounding ellipsoid has been obtained from
+# particles within 2.0 R200c of the halo center
+region = ellipsoid
+region_ellipsoid_matrix[0] = 180.08392, 30.495844, 5.7450113
+region_ellipsoid_matrix[1] = 30.495844, 247.56755, 45.540962
+region_ellipsoid_matrix[2] = 5.7450113, 45.540962, 377.32254
+region_ellipsoid_center    = 0.92963797, 0.3582335, 0.9368965
+
+
+[cosmology]
+Omega_m  = 0.307
+Omega_L  = 0.693
+Omega_b  = 0.04825
+H0       = 67.77
+sigma_8  = 0.8288
+nspec    = 0.9611
+ZeroRadiation = true
+transfer = camb_file
+transfer_file = Eagle_camb_file.txt
+
+[random]
+generator = PANPHASIA
+descriptor =[Panph1,L16,(31250,23438,39063),S12,CH1050187043,EAGLE_L0100_VOL1]
+
+[poisson]
+fft_fine      = yes
+accuracy      = 1e-6
+grad_order    = 6
+laplace_order = 6
+
+[output]
+# TODO: add output options
+""".lstrip()
+
 time_fix_regex = re.compile(r"@[\d\-T:.]+")
 
 
@@ -155,3 +208,16 @@ def test_publication():
     assert time_fix_regex.sub("TIME", output[1]) == time_fix_regex.sub(
         "TIME", reference_output_publication
     )
+
+
+def test_collection():
+    id = "c30de0f3-ab4d-48ad-aa26-f20bb4b70bbd"
+    args = Args(output_path="./", url=DEFAULT_URL, common_directory=True, attempts=1)
+    output = collection_mode(args, id, store=False, traceback_radius=2.0)
+    assert len(output) == 1
+    output = output[0]
+    assert output[0] == "./RefL0100N1504/ics_20770177.cfg"
+
+    assert time_fix_regex.sub("TIME", output[1]) == time_fix_regex.sub(
+        "TIME", reference_output_collection
+    )