From 9384684c01c52b68b203bd7ce1bb0f7903fd421d Mon Sep 17 00:00:00 2001 From: Joel Welling Date: Fri, 14 Aug 2026 14:36:00 -0400 Subject: [PATCH 01/36] introduce CroissantWrapper; add -d; logging; write zip --- .../build_crate_from_dataset.py | 83 ++++++++++++++++--- 1 file changed, 70 insertions(+), 13 deletions(-) diff --git a/src/crate_builder_script/build_crate_from_dataset.py b/src/crate_builder_script/build_crate_from_dataset.py index cfb6e28..d483ebf 100644 --- a/src/crate_builder_script/build_crate_from_dataset.py +++ b/src/crate_builder_script/build_crate_from_dataset.py @@ -1,11 +1,13 @@ import argparse +import json +import logging import os from collections import defaultdict from datetime import datetime, timezone -import logging from pprint import pformat from typing import Any, List +import mlcroissant as mlc import requests from rocrate.model.contextentity import ContextEntity from rocrate.model.person import Person @@ -22,6 +24,7 @@ UUID_API = "https://uuid.api.hubmapconsortium.org" DEFAULT_OUTPUT_PATH = "/tmp/crate_test" +CROISSANT_FILENAME = "croissant.json" # Externally defined identifiers NIH_URI = "https://ror.org/01cwqze88" @@ -33,15 +36,40 @@ AUTH_TOK = os.environ["AUTH_TOK"] +class CroissantWrapper(): + def __init__(self, name: str, description: str): + self.name = name + self.description = description + self.file_objects = [] + self.record_sets = [] + + def add_file(self, file_obj: mlc.FileObject): + self.file_objects.append(file_obj) + + def add_record_set(self, record_set: mlc.RecordSet): + self.record_sets.append(record_set) + + def write(self, croissant_filename: str): + croissant_meta = mlc.Metadata( + id="croissant-spec", + name=self.name, + description=self.description, + distribution=self.file_objects, + record_sets=self.record_sets + ) + with open(croissant_filename, "w", encoding="utf-8") as f: + json.dump(croissant_meta.to_json(), f, indent=2) + def fetch_entity_info(target_id: str) -> dict[str, Any]: - resp = requests.get(ENTITY_API + f"/entities/{target_id}") + resp = requests.get( + ENTITY_API + f"/entities/{target_id}", + headers={"Authorization": f"Bearer {AUTH_TOK}"} + ) resp.raise_for_status() ds_info = resp.json() LOGGER.debug("TOP LEVEL:\n%s", pformat(ds_info, depth=1)) - LOGGER.debug("INGEST METADATA:\n%s", - pformat(ds_info.get("ingest_metadata", {})) - ) + LOGGER.debug("INGEST METADATA:\n%s", pformat(ds_info.get("ingest_metadata", {}))) LOGGER.debug("DIRECT ANCESTORS:\n%s", pformat(ds_info["direct_ancestors"], depth=2)) return ds_info @@ -164,7 +192,7 @@ def build_contributors(crate: ROCrate, contributors: List[dict]) -> List[Context def main() -> None: parser = argparse.ArgumentParser() parser.add_argument( - "target_id", help="Build an RO-Crate for this published dataset" + "target_id", help="Build an RO-Crate for this dataset" ) parser.add_argument( "--outdir", @@ -175,10 +203,21 @@ def main() -> None: ), default=DEFAULT_OUTPUT_PATH, ) + parser.add_argument( + "--debug", + "-d", + action="store_true", + help="Enable debug logging", + ) args = parser.parse_args() target_id = args.target_id outdir = args.outdir + debug = args.debug + if debug: + LOGGER.setLevel(logging.DEBUG) + logging.getLogger("requests").setLevel(logging.DEBUG) + logging.getLogger("urllib3").setLevel(logging.DEBUG) ds_info = fetch_entity_info(target_id) uuid_files = fetch_uuid_files_info(target_id) @@ -190,19 +229,21 @@ def main() -> None: # forbidden as the direct link for a dataset under FAIR. So we can't use the DOI # as the crate root dataset id. crate = ROCrate() + crate.root_dataset["name"] = target_id + crate.root_dataset["description"] = ds_info["title"] + wrapped_croissant = CroissantWrapper(target_id, ds_info["title"]) if "doi_url" in ds_info: doi_url = ds_info["doi_url"] crate.root_dataset["identifier"] = doi_url crate.root_dataset["sameAs"] = doi_url - crate.root_dataset["name"] = target_id - crate.root_dataset["description"] = ds_info["title"] - crate.root_dataset["datePublished"] = str( - datetime.fromtimestamp(ds_info["published_timestamp"] // 1000).astimezone( - timezone.utc + if "published_timestamp" in ds_info: + crate.root_dataset["datePublished"] = str( + datetime.fromtimestamp(ds_info["published_timestamp"] // 1000).astimezone( + timezone.utc + ) ) - ) crate.root_dataset["license"] = crate.add(build_license_entity(crate)) crate.root_dataset["funder"] = crate.add(build_funder_entity(crate)) @@ -212,6 +253,22 @@ def main() -> None: [crate.add(ent) for ent in ent_l] crate.root_dataset["contributor"] = ent_l + if not os.path.isdir(outdir): + os.makedirs(outdir, exist_ok=True) + wrapped_croissant.write(os.path.join(outdir, CROISSANT_FILENAME)) + croissant_crate_file = crate.add_file( + os.path.join(outdir, CROISSANT_FILENAME), + properties={ + "name": "Croissant Metadata Descriptor", + "description": "Machine learning data-loading configurations for this dataset.", + "encodingFormat": "application/ld+json", + "conformsTo": "http://mlcommons.org" + } + ) + + # I don't know why this isn't needed + #crate.root_dataset.append_to("hasPart", croissant_crate_file) + if "files" in ds_info: # This is a derived dataset- include only data products and qa_qc files for fl in ds_info["files"]: @@ -227,7 +284,7 @@ def main() -> None: crate.add_file( asset_url(ds_info["uuid"], fl_blk["path"]), validate_url=True ) - crate.write(outdir) + crate.write_zip(os.path.join(outdir, f"{target_id}_crate.zip")) if __name__ == "__main__": From de508ac088096bca6cb3473e5d250cc2e3d963b9 Mon Sep 17 00:00:00 2001 From: Joel Welling Date: Tue, 18 Aug 2026 14:33:50 -0400 Subject: [PATCH 02/36] separate CroissantWrapper --- .../build_crate_from_dataset.py | 27 ++------------- src/crate_builder_script/croissant_wrapper.py | 33 +++++++++++++++++++ 2 files changed, 35 insertions(+), 25 deletions(-) create mode 100644 src/crate_builder_script/croissant_wrapper.py diff --git a/src/crate_builder_script/build_crate_from_dataset.py b/src/crate_builder_script/build_crate_from_dataset.py index d483ebf..c386b93 100644 --- a/src/crate_builder_script/build_crate_from_dataset.py +++ b/src/crate_builder_script/build_crate_from_dataset.py @@ -7,12 +7,13 @@ from pprint import pformat from typing import Any, List -import mlcroissant as mlc import requests from rocrate.model.contextentity import ContextEntity from rocrate.model.person import Person from rocrate.rocrate import ROCrate +from croissant_wrapper import CroissantWrapper + logging.basicConfig( level=logging.INFO, ) @@ -36,30 +37,6 @@ AUTH_TOK = os.environ["AUTH_TOK"] -class CroissantWrapper(): - def __init__(self, name: str, description: str): - self.name = name - self.description = description - self.file_objects = [] - self.record_sets = [] - - def add_file(self, file_obj: mlc.FileObject): - self.file_objects.append(file_obj) - - def add_record_set(self, record_set: mlc.RecordSet): - self.record_sets.append(record_set) - - def write(self, croissant_filename: str): - croissant_meta = mlc.Metadata( - id="croissant-spec", - name=self.name, - description=self.description, - distribution=self.file_objects, - record_sets=self.record_sets - ) - with open(croissant_filename, "w", encoding="utf-8") as f: - json.dump(croissant_meta.to_json(), f, indent=2) - def fetch_entity_info(target_id: str) -> dict[str, Any]: resp = requests.get( diff --git a/src/crate_builder_script/croissant_wrapper.py b/src/crate_builder_script/croissant_wrapper.py new file mode 100644 index 0000000..817bfcb --- /dev/null +++ b/src/crate_builder_script/croissant_wrapper.py @@ -0,0 +1,33 @@ +import json +import logging +from pprint import pformat + +import mlcroissant as mlc +import requests + +LOGGER = logging.getLogger(__name__) + +class CroissantWrapper(): + def __init__(self, name: str, description: str): + self.name = name + self.description = description + self.file_objects = [] + self.record_sets = [] + + def add_file(self, file_obj: mlc.FileObject): + self.file_objects.append(file_obj) + + def add_record_set(self, record_set: mlc.RecordSet): + self.record_sets.append(record_set) + + def write(self, croissant_filename: str): + croissant_meta = mlc.Metadata( + id="croissant-spec", + name=self.name, + description=self.description, + distribution=self.file_objects, + record_sets=self.record_sets + ) + with open(croissant_filename, "w", encoding="utf-8") as f: + json.dump(croissant_meta.to_json(), f, indent=2) + From abb562112cf2fc8ffe59b963a33d8680f567670f Mon Sep 17 00:00:00 2001 From: Joel Welling Date: Tue, 18 Aug 2026 14:42:43 -0400 Subject: [PATCH 03/36] use a scratch directory; save only zip --- src/crate_builder_script/build_crate_from_dataset.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/src/crate_builder_script/build_crate_from_dataset.py b/src/crate_builder_script/build_crate_from_dataset.py index c386b93..85451fa 100644 --- a/src/crate_builder_script/build_crate_from_dataset.py +++ b/src/crate_builder_script/build_crate_from_dataset.py @@ -6,6 +6,7 @@ from datetime import datetime, timezone from pprint import pformat from typing import Any, List +from tempfile import TemporaryDirectory import requests from rocrate.model.contextentity import ContextEntity @@ -230,11 +231,13 @@ def main() -> None: [crate.add(ent) for ent in ent_l] crate.root_dataset["contributor"] = ent_l + tmpdir = TemporaryDirectory() + if not os.path.isdir(outdir): os.makedirs(outdir, exist_ok=True) - wrapped_croissant.write(os.path.join(outdir, CROISSANT_FILENAME)) + wrapped_croissant.write(os.path.join(tmpdir.name, CROISSANT_FILENAME)) croissant_crate_file = crate.add_file( - os.path.join(outdir, CROISSANT_FILENAME), + os.path.join(tmpdir.name, CROISSANT_FILENAME), properties={ "name": "Croissant Metadata Descriptor", "description": "Machine learning data-loading configurations for this dataset.", @@ -262,7 +265,7 @@ def main() -> None: asset_url(ds_info["uuid"], fl_blk["path"]), validate_url=True ) crate.write_zip(os.path.join(outdir, f"{target_id}_crate.zip")) - + tmpdir.cleanup() if __name__ == "__main__": main() From e68ccdf78235cee14fb52f3ad36ee363af111809 Mon Sep 17 00:00:00 2001 From: Joel Welling Date: Thu, 20 Aug 2026 18:08:39 -0400 Subject: [PATCH 04/36] working on provenance. WIP. --- .../build_crate_from_dataset.py | 23 ++- src/crate_builder_script/croissant_wrapper.py | 169 +++++++++++++++++- src/requirements.txt | 2 +- 3 files changed, 185 insertions(+), 9 deletions(-) diff --git a/src/crate_builder_script/build_crate_from_dataset.py b/src/crate_builder_script/build_crate_from_dataset.py index 85451fa..93da5ef 100644 --- a/src/crate_builder_script/build_crate_from_dataset.py +++ b/src/crate_builder_script/build_crate_from_dataset.py @@ -4,7 +4,7 @@ import os from collections import defaultdict from datetime import datetime, timezone -from pprint import pformat +from pprint import pformat, pprint from typing import Any, List from tempfile import TemporaryDirectory @@ -34,7 +34,7 @@ OBOLIB_URI = "http://purl.obolibrary.org/obo" # TARGET_ID = "HBM567.VCBK.562" -# TARGET_ID = "HBM487.HJZB.546" +# TARGET_ID = "HBM487.HJZB.546" # primary dataset AUTH_TOK = os.environ["AUTH_TOK"] @@ -48,7 +48,17 @@ def fetch_entity_info(target_id: str) -> dict[str, Any]: ds_info = resp.json() LOGGER.debug("TOP LEVEL:\n%s", pformat(ds_info, depth=1)) LOGGER.debug("INGEST METADATA:\n%s", pformat(ds_info.get("ingest_metadata", {}))) - LOGGER.debug("DIRECT ANCESTORS:\n%s", pformat(ds_info["direct_ancestors"], depth=2)) + LOGGER.debug("METADATA:\n%s", pformat(ds_info.get("metadata", {}))) + LOGGER.debug("DIRECT ANCESTORS:\n%s", pformat(ds_info.get("direct_ancestors"), depth=2)) + LOGGER.debug("DIRECT ANCESTOR:\n%s", pformat(ds_info.get("direct_ancestor"), depth=2)) + if "direct_ancestors" in ds_info: + first_ancestor = ds_info["direct_ancestors"][0] + elif "direct_ancestor" in ds_info: + first_ancestor = ds_info["direct_ancestor"] + else: + first_ancestor = {} + LOGGER.debug("ANCESTOR INGEST MD\n%s", pformat(first_ancestor.get("ingest_metadata", {}))) + LOGGER.debug("ANCESTOR MD\n%s", pformat(first_ancestor.get("metadata", {}))) return ds_info @@ -58,7 +68,7 @@ def fetch_uuid_files_info(target_id: str) -> dict[str, Any]: headers={"Authorization": f"Bearer {AUTH_TOK}"}, ) resp.raise_for_status() - LOGGER.debug("UUID FILES first 10:\n%s", pformat(resp.json()[:10])) + # LOGGER.debug("UUID FILES first 10:\n%s", pformat(resp.json()[:10])) return resp.json() @@ -197,6 +207,10 @@ def main() -> None: logging.getLogger("requests").setLevel(logging.DEBUG) logging.getLogger("urllib3").setLevel(logging.DEBUG) ds_info = fetch_entity_info(target_id) + print("TEST") + from pprint import pprint + pprint(fetch_entity_info("HBM976.XLDJ.575"), depth=2) + print("END TEST") uuid_files = fetch_uuid_files_info(target_id) blk_idx = {} @@ -209,6 +223,7 @@ def main() -> None: crate = ROCrate() crate.root_dataset["name"] = target_id crate.root_dataset["description"] = ds_info["title"] + CroissantWrapper.test(ds_info) wrapped_croissant = CroissantWrapper(target_id, ds_info["title"]) if "doi_url" in ds_info: diff --git a/src/crate_builder_script/croissant_wrapper.py b/src/crate_builder_script/croissant_wrapper.py index 817bfcb..4ed98e5 100644 --- a/src/crate_builder_script/croissant_wrapper.py +++ b/src/crate_builder_script/croissant_wrapper.py @@ -3,11 +3,163 @@ from pprint import pformat import mlcroissant as mlc -import requests LOGGER = logging.getLogger(__name__) +HUBMAP = "https://hubmapconsortium.org/" + +def _own_dag(entity: dict) -> list: + return (entity.get("ingest_metadata") or {}).get("dag_provenance_list", []) + + +def is_processed(entity: dict) -> bool: + """Raw vs processed: `creation_action` is 'Create Dataset Activity' vs 'Central Process'.""" + return "process" in (entity.get("creation_action") or "").lower() + + +def _protocol_dois(md: dict) -> list[str]: + dois = [] + for k in ("preparation_protocol_doi", "reagent_prep_protocols_io_doi", + "section_prep_protocols_io_doi"): + v = (md.get(k) or "").strip() + if not v: + continue + dois.append(f"https://dx.doi.org/{v}" if v.startswith("10.") else v) + return dois + + +def _pipeline_steps(dag_list: list) -> list[dict]: + return [{}] +""" + steps, seen = [], set() + for s in dag_list or []: + repo = (s.get("origin") or "").strip().replace(".git", "") + name = repo.rsplit("/", 1)[-1] if repo else (s.get("name") or "") + commit = (s.get("hash") or "")[:7] + cwl = s.get("name") or "" + key = (name, commit, cwl) + if not name or key in seen: + continue + seen.add(key) + steps.append({"name": name, "repo": repo, "commit": commit, "cwl": cwl}) + return steps +""" + +def _acquisition_activity(entity: dict, md: dict) -> dict: + def agent(name, role=None, org=False): + n = {"@type": "prov:Organization" if org else "prov:Person", "schema:name": name} + if role: + n["prov:role"] = role + return n + assoc = [] + if md.get("pi"): + assoc.append(agent(md["pi"], role="principal investigator")) + if md.get("operator"): + assoc.append(agent(md["operator"], role="operator")) + if entity.get("group_name"): + assoc.append(agent(entity["group_name"], org=True)) + act = { + "@type": "prov:Activity", + "schema:name": f"{entity.get('dataset_type', 'assay')} acquisition", + "hubmap:instrument": " ".join(filter(None, [md.get("acquisition_instrument_vendor"), + md.get("acquisition_instrument_model")])), + "hubmap:numberOfAntibodies": md.get("number_of_antibodies"), + "hubmap:numberOfImagingRounds": md.get("number_of_biomarker_imaging_rounds"), + "hubmap:numberOfChannels": md.get("number_of_channels"), + "prov:wasAssociatedWith": assoc, + } + if md.get("execution_datetime"): + act["prov:startedAtTime"] = md["execution_datetime"] + protocols = [{"@type": ["prov:Entity", "schema:CreativeWork"], "@id": d, "prov:role": "protocol"} + for d in _protocol_dois(md)] + if protocols: + act["prov:used"] = protocols + return {k: v for k, v in act.items() if v not in (None, "", [])} + + +def _specimen_chain(context: dict) -> dict: + def node(anc): + n = {"@type": "prov:Entity", "@id": HUBMAP + (anc.get("hubmap_id") or ""), + "schema:name": anc.get("hubmap_id"), "hubmap:entityType": anc.get("entity_type"), + "hubmap:sampleCategory": anc.get("sample_category")} + rui = anc.get("rui_location") + if rui: + r = json.loads(rui) if isinstance(rui, str) else rui + n["hubmap:ccfAnnotations"] = r.get("ccf_annotations") + n["hubmap:dimensions"] = {"x": r.get("x_dimension"), "y": r.get("y_dimension"), + "z": r.get("z_dimension"), "unit": r.get("dimension_units")} + return {k: v for k, v in n.items() if v not in (None, "", [])} + order = {"section": 0, "block": 1, "organ": 2} + ancs = [a for a in context.get("ancestors", []) if a.get("entity_type") in ("Sample", "Donor")] + ancs.sort(key=lambda a: order.get(a.get("sample_category"), 4)) + derived = None + for anc in reversed(ancs): + n = node(anc) + if derived: + n["prov:wasDerivedFrom"] = derived + derived = n + return derived + +def _pipeline_activity(dag_list: list) -> dict: + return {} +""" + agents = [] + for st in _pipeline_steps(dag_list): + a = {"@type": ["prov:SoftwareAgent", "schema:SoftwareApplication"], + "schema:name": st["name"] + (f" [{st['cwl']}]" if st["cwl"] else ""), + "schema:codeRepository": st["repo"], "hubmap:commit": st["commit"]} + agents.append({k: v for k, v in a.items() if v}) + act = {"@type": "prov:Activity", "schema:name": "HuBMAP uniform processing pipeline"} + if agents: + act["prov:wasAssociatedWith"] = agents + return act +""" + +def build_embedded_provenance(entity, context, md, descendants=None, raw_entity=None, raw_md=None) -> dict: + """ + PROCESSED subject: wasGeneratedBy its own pipeline; wasDerivedFrom the raw parent + (which carries the acquisition activity + specimen chain). RAW subject: wasGeneratedBy + acquisition; wasDerivedFrom specimen chain; + a light forward pointer to processed versions. + """ + if is_processed(entity) and raw_entity is not None: + raw_node = { + "@type": "prov:Entity", "@id": HUBMAP + (raw_entity.get("hubmap_id") or ""), + "schema:name": raw_entity.get("hubmap_id"), + "hubmap:datasetType": raw_entity.get("dataset_type"), + "prov:wasGeneratedBy": _acquisition_activity(raw_entity, raw_md or {}), + "prov:wasDerivedFrom": _specimen_chain(context) + } + #raw_node = {k: v for k, v in raw_node.items() if v} + return {"prov:wasGeneratedBy": _pipeline_activity(_own_dag(entity)), + "prov:wasDerivedFrom": raw_node} + provo = {"prov:wasGeneratedBy": _acquisition_activity(entity, md)} + if chain := _specimen_chain(context): + provo["prov:wasDerivedFrom"] = chain + if descendants: + provo["hubmap:hasProcessedDataset"] = [ + {"@type": "prov:Entity", "@id": HUBMAP + (d.get("hubmap_id") or ""), + "schema:name": d.get("hubmap_id"), "hubmap:datasetType": d.get("dataset_type")} + for d in descendants] + return provo + + class CroissantWrapper(): + @classmethod + def test(cls, entity_dict: dict) -> None: + ancestors = entity_dict.get("direct_ancestors", []) + parent_dict = ancestors[0] if len(ancestors) == 1 else None + LOGGER.info("Testing CroissantWrapper:\n%s", + pformat( + build_embedded_provenance( + entity_dict, + {}, # context + entity_dict.get("metadata"), # md + descendants=entity_dict.get("direct_descendants", []), + raw_entity=parent_dict, + raw_md=parent_dict.get("metadata") if parent_dict else None + ) + )) + def __init__(self, name: str, description: str): self.name = name self.description = description @@ -26,8 +178,17 @@ def write(self, croissant_filename: str): name=self.name, description=self.description, distribution=self.file_objects, - record_sets=self.record_sets - ) + record_sets=self.record_sets, + ctx=mlc.Context( + is_live_dataset=False + ) + ).to_json() + croissant_meta["@context"].update({ + "hubmap": "https://hubmapconsortium.org/", + "mlc": "https://mlcommons.org/", + "prov": "http://www.w3.org/ns/prov#", + "schema": "http://schema.org/" + }), with open(croissant_filename, "w", encoding="utf-8") as f: - json.dump(croissant_meta.to_json(), f, indent=2) + json.dump(croissant_meta, f, indent=2) diff --git a/src/requirements.txt b/src/requirements.txt index d8d9c6d..d833963 100644 --- a/src/requirements.txt +++ b/src/requirements.txt @@ -4,6 +4,6 @@ PyYAML==6.0.2 requests==2.32.3 rocrate>=0.15.0 bagit -mlcroissant +mlcroissant[dev]>=1.1.0 git+https://github.com/hubmapconsortium/cwltool.git@docker-gpu#egg=cwltool setuptools<82 From 9c05578e916751930b1bb502ce1d7e05b583f688 Mon Sep 17 00:00:00 2001 From: Joel Welling Date: Fri, 21 Aug 2026 17:28:41 -0400 Subject: [PATCH 05/36] walk_ancestors is working. WIP. --- src/crate_builder_script/api_calls.py | 52 ++++++++++++++++ .../build_crate_from_dataset.py | 61 +++---------------- src/crate_builder_script/croissant_wrapper.py | 60 ++++++++++++++++-- 3 files changed, 116 insertions(+), 57 deletions(-) create mode 100644 src/crate_builder_script/api_calls.py diff --git a/src/crate_builder_script/api_calls.py b/src/crate_builder_script/api_calls.py new file mode 100644 index 0000000..e5f4605 --- /dev/null +++ b/src/crate_builder_script/api_calls.py @@ -0,0 +1,52 @@ +import logging +import os +from pprint import pformat, pprint +from typing import Any + +import requests + +LOGGER = logging.getLogger(__name__) + +ENTITY_API = "https://entity.api.hubmapconsortium.org" +ASSETS_API = "https://assets.hubmapconsortium.org" +UUID_API = "https://uuid.api.hubmapconsortium.org" + +AUTH_TOK = os.environ["AUTH_TOK"] + + +def fetch_entity_info(target_id: str) -> dict[str, Any]: + resp = requests.get( + ENTITY_API + f"/entities/{target_id}", + headers={"Authorization": f"Bearer {AUTH_TOK}"} + ) + resp.raise_for_status() + ds_info = resp.json() + LOGGER.debug("TOP LEVEL for %s:\n%s", target_id, pformat(ds_info, depth=1)) + LOGGER.debug("INGEST METADATA:\n%s", pformat(ds_info.get("ingest_metadata", {}), + depth=2)) + LOGGER.debug("METADATA:\n%s", pformat(ds_info.get("metadata", {}), depth=2)) + LOGGER.debug("DIRECT ANCESTORS:\n%s", pformat(ds_info.get("direct_ancestors"), depth=2)) + LOGGER.debug("DIRECT ANCESTOR:\n%s", pformat(ds_info.get("direct_ancestor"), depth=2)) + if "direct_ancestors" in ds_info: + first_ancestor = ds_info["direct_ancestors"][0] + elif "direct_ancestor" in ds_info: + first_ancestor = ds_info["direct_ancestor"] + else: + first_ancestor = {} + LOGGER.debug("ANCESTOR INGEST MD\n%s", pformat(first_ancestor.get("ingest_metadata", {}))) + LOGGER.debug("ANCESTOR MD\n%s", pformat(first_ancestor.get("metadata", {}))) + return ds_info + + +def fetch_uuid_files_info(target_id: str) -> dict[str, Any]: + resp = requests.get( + UUID_API + f"/{target_id}/files", + headers={"Authorization": f"Bearer {AUTH_TOK}"}, + ) + resp.raise_for_status() + # LOGGER.debug("UUID FILES first 10:\n%s", pformat(resp.json()[:10])) + return resp.json() + + +def asset_url(uuid: str, rel_path: str) -> str: + return f"{ASSETS_API}/{uuid}/{rel_path}" diff --git a/src/crate_builder_script/build_crate_from_dataset.py b/src/crate_builder_script/build_crate_from_dataset.py index 93da5ef..412e932 100644 --- a/src/crate_builder_script/build_crate_from_dataset.py +++ b/src/crate_builder_script/build_crate_from_dataset.py @@ -8,11 +8,11 @@ from typing import Any, List from tempfile import TemporaryDirectory -import requests from rocrate.model.contextentity import ContextEntity from rocrate.model.person import Person from rocrate.rocrate import ROCrate +import api_calls from croissant_wrapper import CroissantWrapper logging.basicConfig( @@ -21,10 +21,6 @@ LOGGER = logging.getLogger(__name__) -ENTITY_API = "https://entity.api.hubmapconsortium.org" -ASSETS_API = "https://assets.hubmapconsortium.org" -UUID_API = "https://uuid.api.hubmapconsortium.org" - DEFAULT_OUTPUT_PATH = "/tmp/crate_test" CROISSANT_FILENAME = "croissant.json" @@ -36,45 +32,6 @@ # TARGET_ID = "HBM567.VCBK.562" # TARGET_ID = "HBM487.HJZB.546" # primary dataset -AUTH_TOK = os.environ["AUTH_TOK"] - - -def fetch_entity_info(target_id: str) -> dict[str, Any]: - resp = requests.get( - ENTITY_API + f"/entities/{target_id}", - headers={"Authorization": f"Bearer {AUTH_TOK}"} - ) - resp.raise_for_status() - ds_info = resp.json() - LOGGER.debug("TOP LEVEL:\n%s", pformat(ds_info, depth=1)) - LOGGER.debug("INGEST METADATA:\n%s", pformat(ds_info.get("ingest_metadata", {}))) - LOGGER.debug("METADATA:\n%s", pformat(ds_info.get("metadata", {}))) - LOGGER.debug("DIRECT ANCESTORS:\n%s", pformat(ds_info.get("direct_ancestors"), depth=2)) - LOGGER.debug("DIRECT ANCESTOR:\n%s", pformat(ds_info.get("direct_ancestor"), depth=2)) - if "direct_ancestors" in ds_info: - first_ancestor = ds_info["direct_ancestors"][0] - elif "direct_ancestor" in ds_info: - first_ancestor = ds_info["direct_ancestor"] - else: - first_ancestor = {} - LOGGER.debug("ANCESTOR INGEST MD\n%s", pformat(first_ancestor.get("ingest_metadata", {}))) - LOGGER.debug("ANCESTOR MD\n%s", pformat(first_ancestor.get("metadata", {}))) - return ds_info - - -def fetch_uuid_files_info(target_id: str) -> dict[str, Any]: - resp = requests.get( - UUID_API + f"/{target_id}/files", - headers={"Authorization": f"Bearer {AUTH_TOK}"}, - ) - resp.raise_for_status() - # LOGGER.debug("UUID FILES first 10:\n%s", pformat(resp.json()[:10])) - return resp.json() - - -def asset_url(uuid: str, rel_path: str) -> str: - return f"{ASSETS_API}/{uuid}/{rel_path}" - def build_funder_entity(crate: ROCrate) -> ContextEntity: funder_props = { @@ -206,13 +163,11 @@ def main() -> None: LOGGER.setLevel(logging.DEBUG) logging.getLogger("requests").setLevel(logging.DEBUG) logging.getLogger("urllib3").setLevel(logging.DEBUG) - ds_info = fetch_entity_info(target_id) - print("TEST") - from pprint import pprint - pprint(fetch_entity_info("HBM976.XLDJ.575"), depth=2) - print("END TEST") + logging.getLogger("api_calls").setLevel(logging.DEBUG) + logging.getLogger("croissant_wrapper").setLevel(logging.DEBUG) + ds_info = api_calls.fetch_entity_info(target_id) - uuid_files = fetch_uuid_files_info(target_id) + uuid_files = api_calls.fetch_uuid_files_info(target_id) blk_idx = {} for file_blk in uuid_files: blk_idx[file_blk["path"]] = file_blk @@ -270,14 +225,16 @@ def main() -> None: if fl["is_data_product"] or fl["is_qa_qc"]: LOGGER.debug(f"Adding {fl['rel_path']}") crate.add_file( - asset_url(ds_info["uuid"], fl["rel_path"]), validate_url=True + api_calls.asset_url(ds_info["uuid"], fl["rel_path"]), + validate_url=True ) else: LOGGER.debug(f"{fl['rel_path']} is not a data product") else: for fl_blk in blk_idx.values(): crate.add_file( - asset_url(ds_info["uuid"], fl_blk["path"]), validate_url=True + api_calls.asset_url(ds_info["uuid"], fl_blk["path"]), + validate_url=True ) crate.write_zip(os.path.join(outdir, f"{target_id}_crate.zip")) tmpdir.cleanup() diff --git a/src/crate_builder_script/croissant_wrapper.py b/src/crate_builder_script/croissant_wrapper.py index 4ed98e5..4643aef 100644 --- a/src/crate_builder_script/croissant_wrapper.py +++ b/src/crate_builder_script/croissant_wrapper.py @@ -1,9 +1,13 @@ import json import logging -from pprint import pformat +from os import walk +from pprint import pformat, pprint +from venv import logger import mlcroissant as mlc +from api_calls import fetch_entity_info + LOGGER = logging.getLogger(__name__) HUBMAP = "https://hubmapconsortium.org/" @@ -77,7 +81,45 @@ def agent(name, role=None, org=False): return {k: v for k, v in act.items() if v not in (None, "", [])} -def _specimen_chain(context: dict) -> dict: +def walk_ancestors(entity: dict) -> list[dict]: + """Return a list of ancestor entities, starting with the immediate parent.""" + rslt = [] + e_type = entity.get("entity_type") + e_id = entity.get("hubmap_id") + LOGGER.debug(f"walk_ancestors {e_id} {e_type}") + if e_type == "Dataset": + ancs = [walk_ancestors(anc) for anc in entity.get("direct_ancestors", [])] + if ancs: + rslt.append((e_type, e_id, ancs)) + else: + md = entity.get("metadata", {}) + if "parent_sample_id" in md: + samp_id = md["parent_sample_id"] + samp_entity = fetch_entity_info(samp_id) + rslt.append((e_type, e_id, walk_ancestors(samp_entity))) + elif e_type in ("Sample", "Donor"): + e_cat = entity.get("sample_category", "UNKNOWN SAMPLE CATEGORY") + LOGGER.debug(f"walk_ancestors sample category is {e_cat}") + if "direct_ancestor" not in entity: + LOGGER.debug(f"walk_ancestors fetching dead-end sample {e_id}") + entity = fetch_entity_info(e_id) + LOGGER.debug("walk_ancestors fetch yielded:\n%s", + pformat(entity, depth=2)) + LOGGER.debug("walk_ancestors end of walk jump result") + new_entity = entity.get("direct_ancestor", {}) + if e_type == "Donor": + rslt.append((f"{e_type}", e_id, None)) + else: + rslt.append((f"{e_type} {e_cat}", e_id, walk_ancestors(new_entity))) + else: + LOGGER.warning(f"walk_ancestors UNKNOWN ETYPE {e_type} for {e_id}") + return rslt + + +def _specimen_chain(entity: dict, recur=0) -> dict: + if recur > 10: + LOGGER.warning("Recursion limit reached in _specimen_chain for entity %s", entity.get("hubmap_id")) + return {} def node(anc): n = {"@type": "prov:Entity", "@id": HUBMAP + (anc.get("hubmap_id") or ""), "schema:name": anc.get("hubmap_id"), "hubmap:entityType": anc.get("entity_type"), @@ -90,6 +132,12 @@ def node(anc): "z": r.get("z_dimension"), "unit": r.get("dimension_units")} return {k: v for k, v in n.items() if v not in (None, "", [])} order = {"section": 0, "block": 1, "organ": 2} + anc_entities = walk_ancestors(entity) + LOGGER.info("Ancestor entities for %s: %s", entity.get("hubmap_id"), anc_entities) + return None + """ while "direct_ancestors" in entity and entity["direct_ancestors"]: + parent_entity = entity["direct_ancestors"][0] + ancs.append(entity) ancs = [a for a in context.get("ancestors", []) if a.get("entity_type") in ("Sample", "Donor")] ancs.sort(key=lambda a: order.get(a.get("sample_category"), 4)) derived = None @@ -99,7 +147,7 @@ def node(anc): n["prov:wasDerivedFrom"] = derived derived = n return derived - + """ def _pipeline_activity(dag_list: list) -> dict: return {} """ @@ -121,19 +169,21 @@ def build_embedded_provenance(entity, context, md, descendants=None, raw_entity= (which carries the acquisition activity + specimen chain). RAW subject: wasGeneratedBy acquisition; wasDerivedFrom specimen chain; + a light forward pointer to processed versions. """ + test_chain = walk_ancestors(entity) + LOGGER.info("walk_ancestors for %s: %s", entity.get("hubmap_id"), test_chain) if is_processed(entity) and raw_entity is not None: raw_node = { "@type": "prov:Entity", "@id": HUBMAP + (raw_entity.get("hubmap_id") or ""), "schema:name": raw_entity.get("hubmap_id"), "hubmap:datasetType": raw_entity.get("dataset_type"), "prov:wasGeneratedBy": _acquisition_activity(raw_entity, raw_md or {}), - "prov:wasDerivedFrom": _specimen_chain(context) + "prov:wasDerivedFrom": _specimen_chain(raw_entity) } #raw_node = {k: v for k, v in raw_node.items() if v} return {"prov:wasGeneratedBy": _pipeline_activity(_own_dag(entity)), "prov:wasDerivedFrom": raw_node} provo = {"prov:wasGeneratedBy": _acquisition_activity(entity, md)} - if chain := _specimen_chain(context): + if chain := _specimen_chain(entity): provo["prov:wasDerivedFrom"] = chain if descendants: provo["hubmap:hasProcessedDataset"] = [ From 18197c5b57cf4c404f126b321b334f0028565c6e Mon Sep 17 00:00:00 2001 From: Joel Welling Date: Tue, 25 Aug 2026 17:45:25 -0400 Subject: [PATCH 06/36] working walk_ancestors --- src/crate_builder_script/api_calls.py | 49 ++++++++++++++++- src/crate_builder_script/croissant_wrapper.py | 55 +++++-------------- 2 files changed, 62 insertions(+), 42 deletions(-) diff --git a/src/crate_builder_script/api_calls.py b/src/crate_builder_script/api_calls.py index e5f4605..98f6a23 100644 --- a/src/crate_builder_script/api_calls.py +++ b/src/crate_builder_script/api_calls.py @@ -1,7 +1,8 @@ import logging import os -from pprint import pformat, pprint +from pprint import pformat from typing import Any +from collections.abc import Callable import requests @@ -50,3 +51,49 @@ def fetch_uuid_files_info(target_id: str) -> dict[str, Any]: def asset_url(uuid: str, rel_path: str) -> str: return f"{ASSETS_API}/{uuid}/{rel_path}" + + +def walk_ancestors( + entity: dict, + continue_test: Callable[[dict], bool] = lambda ent: True + ) -> list[tuple]: + """Return a list of ancestor entities, starting with the immediate parent.""" + rslt = [] + if not continue_test(entity): + return rslt + e_type = entity.get("entity_type") + e_id = entity.get("hubmap_id") + LOGGER.debug(f"walk_ancestors {e_id} {e_type}") + if e_type == "Dataset": + ancs = [walk_ancestors(anc, continue_test) + for anc in entity.get("direct_ancestors", [])] + if ancs: + all_tuples = [] + for sub_list in ancs: + assert isinstance(sub_list, list) + all_tuples.extend(sub_list) + rslt.append((e_id, entity, all_tuples)) + else: + md = entity.get("metadata", {}) + if "parent_sample_id" in md: + # The parent is a sample + samp_id = md["parent_sample_id"] + samp_entity = fetch_entity_info(samp_id) + rslt.append((e_id, entity, walk_ancestors(samp_entity, continue_test))) + elif e_type in ("Sample", "Donor"): + e_cat = entity.get("sample_category", "UNKNOWN SAMPLE CATEGORY") + LOGGER.debug(f"walk_ancestors sample category is {e_cat}") + if "direct_ancestor" not in entity: + LOGGER.debug(f"walk_ancestors fetching dead-end sample {e_id}") + entity = fetch_entity_info(e_id) + LOGGER.debug("walk_ancestors fetch yielded:\n%s", + pformat(entity, depth=2)) + LOGGER.debug("walk_ancestors end of walk jump result") + new_entity = entity.get("direct_ancestor", {}) + if e_type == "Donor": + rslt.append((e_id, entity, None)) + else: + rslt.append((e_id, entity, walk_ancestors(new_entity, continue_test))) + else: + LOGGER.warning(f"walk_ancestors UNKNOWN ETYPE {e_type} for {e_id}") + return rslt diff --git a/src/crate_builder_script/croissant_wrapper.py b/src/crate_builder_script/croissant_wrapper.py index 4643aef..18558e1 100644 --- a/src/crate_builder_script/croissant_wrapper.py +++ b/src/crate_builder_script/croissant_wrapper.py @@ -2,16 +2,16 @@ import logging from os import walk from pprint import pformat, pprint -from venv import logger import mlcroissant as mlc -from api_calls import fetch_entity_info +from api_calls import fetch_entity_info, walk_ancestors LOGGER = logging.getLogger(__name__) HUBMAP = "https://hubmapconsortium.org/" + def _own_dag(entity: dict) -> list: return (entity.get("ingest_metadata") or {}).get("dag_provenance_list", []) @@ -81,41 +81,6 @@ def agent(name, role=None, org=False): return {k: v for k, v in act.items() if v not in (None, "", [])} -def walk_ancestors(entity: dict) -> list[dict]: - """Return a list of ancestor entities, starting with the immediate parent.""" - rslt = [] - e_type = entity.get("entity_type") - e_id = entity.get("hubmap_id") - LOGGER.debug(f"walk_ancestors {e_id} {e_type}") - if e_type == "Dataset": - ancs = [walk_ancestors(anc) for anc in entity.get("direct_ancestors", [])] - if ancs: - rslt.append((e_type, e_id, ancs)) - else: - md = entity.get("metadata", {}) - if "parent_sample_id" in md: - samp_id = md["parent_sample_id"] - samp_entity = fetch_entity_info(samp_id) - rslt.append((e_type, e_id, walk_ancestors(samp_entity))) - elif e_type in ("Sample", "Donor"): - e_cat = entity.get("sample_category", "UNKNOWN SAMPLE CATEGORY") - LOGGER.debug(f"walk_ancestors sample category is {e_cat}") - if "direct_ancestor" not in entity: - LOGGER.debug(f"walk_ancestors fetching dead-end sample {e_id}") - entity = fetch_entity_info(e_id) - LOGGER.debug("walk_ancestors fetch yielded:\n%s", - pformat(entity, depth=2)) - LOGGER.debug("walk_ancestors end of walk jump result") - new_entity = entity.get("direct_ancestor", {}) - if e_type == "Donor": - rslt.append((f"{e_type}", e_id, None)) - else: - rslt.append((f"{e_type} {e_cat}", e_id, walk_ancestors(new_entity))) - else: - LOGGER.warning(f"walk_ancestors UNKNOWN ETYPE {e_type} for {e_id}") - return rslt - - def _specimen_chain(entity: dict, recur=0) -> dict: if recur > 10: LOGGER.warning("Recursion limit reached in _specimen_chain for entity %s", entity.get("hubmap_id")) @@ -132,8 +97,6 @@ def node(anc): "z": r.get("z_dimension"), "unit": r.get("dimension_units")} return {k: v for k, v in n.items() if v not in (None, "", [])} order = {"section": 0, "block": 1, "organ": 2} - anc_entities = walk_ancestors(entity) - LOGGER.info("Ancestor entities for %s: %s", entity.get("hubmap_id"), anc_entities) return None """ while "direct_ancestors" in entity and entity["direct_ancestors"]: parent_entity = entity["direct_ancestors"][0] @@ -169,8 +132,6 @@ def build_embedded_provenance(entity, context, md, descendants=None, raw_entity= (which carries the acquisition activity + specimen chain). RAW subject: wasGeneratedBy acquisition; wasDerivedFrom specimen chain; + a light forward pointer to processed versions. """ - test_chain = walk_ancestors(entity) - LOGGER.info("walk_ancestors for %s: %s", entity.get("hubmap_id"), test_chain) if is_processed(entity) and raw_entity is not None: raw_node = { "@type": "prov:Entity", "@id": HUBMAP + (raw_entity.get("hubmap_id") or ""), @@ -196,6 +157,18 @@ def build_embedded_provenance(entity, context, md, descendants=None, raw_entity= class CroissantWrapper(): @classmethod def test(cls, entity_dict: dict) -> None: + def walk_datasets_only(d: dict) -> bool: + return (d["entity_type"] == "Dataset") + def anc_summary_str(anc_chain: list) -> str: + for id, ent, sub_list in anc_chain: + if sub_list is None: + return f"{id} {ent['entity_type']} none" + else: + return f"{id} {ent['entity_type']} {{...}} {'[' + ' '.join(anc_summary_str([elt]) for elt in sub_list) + ']'}" + test_chain = walk_ancestors(entity_dict, walk_datasets_only) + LOGGER.info("walk_ancestors for %s: [%s]", + entity_dict.get("hubmap_id"), + anc_summary_str(test_chain)) ancestors = entity_dict.get("direct_ancestors", []) parent_dict = ancestors[0] if len(ancestors) == 1 else None LOGGER.info("Testing CroissantWrapper:\n%s", From 08620e9890be10db21d440e1bb4a633fdc795dcb Mon Sep 17 00:00:00 2001 From: Joel Welling Date: Tue, 25 Aug 2026 18:11:11 -0400 Subject: [PATCH 07/36] better docstring for walk_ancestors --- src/crate_builder_script/api_calls.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/src/crate_builder_script/api_calls.py b/src/crate_builder_script/api_calls.py index 98f6a23..d22fc7e 100644 --- a/src/crate_builder_script/api_calls.py +++ b/src/crate_builder_script/api_calls.py @@ -57,7 +57,15 @@ def walk_ancestors( entity: dict, continue_test: Callable[[dict], bool] = lambda ent: True ) -> list[tuple]: - """Return a list of ancestor entities, starting with the immediate parent.""" + """ + Given an entity dictionary, return a list of tuples. Each tuple has + the form: + (hubmap_id entity_dict list-of-ancestors) + where list-of-ancestors is None or a list of tuples of the same form. + + continue_test takes an entity dict as a parameter and returns True if + the descent should continue to the children of that entity, False otherwise. + """ rslt = [] if not continue_test(entity): return rslt From bc6b25f6f113be9097e9ff29f2186e6785894bc8 Mon Sep 17 00:00:00 2001 From: Joel Welling Date: Wed, 26 Aug 2026 11:12:01 -0400 Subject: [PATCH 08/36] croissant provenance fully functional --- src/crate_builder_script/api_calls.py | 12 +++ .../build_crate_from_dataset.py | 1 - src/crate_builder_script/croissant_wrapper.py | 87 +++++++++---------- 3 files changed, 53 insertions(+), 47 deletions(-) diff --git a/src/crate_builder_script/api_calls.py b/src/crate_builder_script/api_calls.py index d22fc7e..8ca3d02 100644 --- a/src/crate_builder_script/api_calls.py +++ b/src/crate_builder_script/api_calls.py @@ -105,3 +105,15 @@ def walk_ancestors( else: LOGGER.warning(f"walk_ancestors UNKNOWN ETYPE {e_type} for {e_id}") return rslt + + +def listify(ancestor_chain: list, omit_test: Callable[[dict], bool]) -> list: + assert len(ancestor_chain) == 1, "listify must start on a 1-tuple chain" + hubmap_id, entity_dict, ancestors = ancestor_chain[0] + rslt = [] + if not omit_test(entity_dict): + rslt.append(entity_dict) + if ancestors: + for anc in ancestors: + rslt.extend(listify([anc], omit_test)) + return rslt diff --git a/src/crate_builder_script/build_crate_from_dataset.py b/src/crate_builder_script/build_crate_from_dataset.py index 412e932..309199c 100644 --- a/src/crate_builder_script/build_crate_from_dataset.py +++ b/src/crate_builder_script/build_crate_from_dataset.py @@ -178,7 +178,6 @@ def main() -> None: crate = ROCrate() crate.root_dataset["name"] = target_id crate.root_dataset["description"] = ds_info["title"] - CroissantWrapper.test(ds_info) wrapped_croissant = CroissantWrapper(target_id, ds_info["title"]) if "doi_url" in ds_info: diff --git a/src/crate_builder_script/croissant_wrapper.py b/src/crate_builder_script/croissant_wrapper.py index 18558e1..100c724 100644 --- a/src/crate_builder_script/croissant_wrapper.py +++ b/src/crate_builder_script/croissant_wrapper.py @@ -5,7 +5,7 @@ import mlcroissant as mlc -from api_calls import fetch_entity_info, walk_ancestors +from api_calls import fetch_entity_info, walk_ancestors, listify LOGGER = logging.getLogger(__name__) @@ -33,8 +33,6 @@ def _protocol_dois(md: dict) -> list[str]: def _pipeline_steps(dag_list: list) -> list[dict]: - return [{}] -""" steps, seen = [], set() for s in dag_list or []: repo = (s.get("origin") or "").strip().replace(".git", "") @@ -47,7 +45,6 @@ def _pipeline_steps(dag_list: list) -> list[dict]: seen.add(key) steps.append({"name": name, "repo": repo, "commit": commit, "cwl": cwl}) return steps -""" def _acquisition_activity(entity: dict, md: dict) -> dict: def agent(name, role=None, org=False): @@ -82,9 +79,6 @@ def agent(name, role=None, org=False): def _specimen_chain(entity: dict, recur=0) -> dict: - if recur > 10: - LOGGER.warning("Recursion limit reached in _specimen_chain for entity %s", entity.get("hubmap_id")) - return {} def node(anc): n = {"@type": "prov:Entity", "@id": HUBMAP + (anc.get("hubmap_id") or ""), "schema:name": anc.get("hubmap_id"), "hubmap:entityType": anc.get("entity_type"), @@ -96,12 +90,11 @@ def node(anc): n["hubmap:dimensions"] = {"x": r.get("x_dimension"), "y": r.get("y_dimension"), "z": r.get("z_dimension"), "unit": r.get("dimension_units")} return {k: v for k, v in n.items() if v not in (None, "", [])} + ancs = listify( + walk_ancestors(entity), + omit_test=lambda dct: dct["entity_type"]=="Dataset" + ) order = {"section": 0, "block": 1, "organ": 2} - return None - """ while "direct_ancestors" in entity and entity["direct_ancestors"]: - parent_entity = entity["direct_ancestors"][0] - ancs.append(entity) - ancs = [a for a in context.get("ancestors", []) if a.get("entity_type") in ("Sample", "Donor")] ancs.sort(key=lambda a: order.get(a.get("sample_category"), 4)) derived = None for anc in reversed(ancs): @@ -110,10 +103,9 @@ def node(anc): n["prov:wasDerivedFrom"] = derived derived = n return derived - """ + + def _pipeline_activity(dag_list: list) -> dict: - return {} -""" agents = [] for st in _pipeline_steps(dag_list): a = {"@type": ["prov:SoftwareAgent", "schema:SoftwareApplication"], @@ -124,15 +116,26 @@ def _pipeline_activity(dag_list: list) -> dict: if agents: act["prov:wasAssociatedWith"] = agents return act -""" -def build_embedded_provenance(entity, context, md, descendants=None, raw_entity=None, raw_md=None) -> dict: + +def build_embedded_provenance(entity, md, descendants=None) -> dict: """ PROCESSED subject: wasGeneratedBy its own pipeline; wasDerivedFrom the raw parent (which carries the acquisition activity + specimen chain). RAW subject: wasGeneratedBy acquisition; wasDerivedFrom specimen chain; + a light forward pointer to processed versions. """ - if is_processed(entity) and raw_entity is not None: + if is_processed(entity): + hubmap_id = entity["hubmap_id"] + ancestor_chain = walk_ancestors( + entity, + lambda d: d["entity_type"] == "Dataset" + ) + assert len(ancestor_chain) == 1, "internal error walking ancestors" + check_id, ignored_entity, ancestors = ancestor_chain[0] + assert check_id == hubmap_id + assert len(ancestors) == 1, f"Dataset {hubmap_id} has too many ancestors" + raw_id, raw_entity, ignored = ancestors[0] + raw_md = raw_entity.get("metadata") raw_node = { "@type": "prov:Entity", "@id": HUBMAP + (raw_entity.get("hubmap_id") or ""), "schema:name": raw_entity.get("hubmap_id"), @@ -140,7 +143,6 @@ def build_embedded_provenance(entity, context, md, descendants=None, raw_entity= "prov:wasGeneratedBy": _acquisition_activity(raw_entity, raw_md or {}), "prov:wasDerivedFrom": _specimen_chain(raw_entity) } - #raw_node = {k: v for k, v in raw_node.items() if v} return {"prov:wasGeneratedBy": _pipeline_activity(_own_dag(entity)), "prov:wasDerivedFrom": raw_node} provo = {"prov:wasGeneratedBy": _acquisition_activity(entity, md)} @@ -157,31 +159,16 @@ def build_embedded_provenance(entity, context, md, descendants=None, raw_entity= class CroissantWrapper(): @classmethod def test(cls, entity_dict: dict) -> None: - def walk_datasets_only(d: dict) -> bool: - return (d["entity_type"] == "Dataset") - def anc_summary_str(anc_chain: list) -> str: - for id, ent, sub_list in anc_chain: - if sub_list is None: - return f"{id} {ent['entity_type']} none" - else: - return f"{id} {ent['entity_type']} {{...}} {'[' + ' '.join(anc_summary_str([elt]) for elt in sub_list) + ']'}" - test_chain = walk_ancestors(entity_dict, walk_datasets_only) - LOGGER.info("walk_ancestors for %s: [%s]", - entity_dict.get("hubmap_id"), - anc_summary_str(test_chain)) - ancestors = entity_dict.get("direct_ancestors", []) - parent_dict = ancestors[0] if len(ancestors) == 1 else None - LOGGER.info("Testing CroissantWrapper:\n%s", - pformat( - build_embedded_provenance( - entity_dict, - {}, # context - entity_dict.get("metadata"), # md - descendants=entity_dict.get("direct_descendants", []), - raw_entity=parent_dict, - raw_md=parent_dict.get("metadata") if parent_dict else None - ) - )) + LOGGER.info( + "Testing CroissantWrapper:\n%s", + pformat( + build_embedded_provenance( + entity_dict, + md=entity_dict.get("metadata"), + descendants=entity_dict.get("direct_descendants", []) + ) + ) + ) def __init__(self, name: str, description: str): self.name = name @@ -202,7 +189,7 @@ def write(self, croissant_filename: str): description=self.description, distribution=self.file_objects, record_sets=self.record_sets, - ctx=mlc.Context( + ctx=mlc.Context( is_live_dataset=False ) ).to_json() @@ -211,7 +198,15 @@ def write(self, croissant_filename: str): "mlc": "https://mlcommons.org/", "prov": "http://www.w3.org/ns/prov#", "schema": "http://schema.org/" - }), + }) + entity_dict = fetch_entity_info(self.name) + croissant_meta.update( + build_embedded_provenance( + entity_dict, + md=entity_dict.get("metadata"), # md + descendants=entity_dict.get("direct_descendants", []) + ) + ) with open(croissant_filename, "w", encoding="utf-8") as f: json.dump(croissant_meta, f, indent=2) From 9569375d650158341c009d641f2a5becb7954fc7 Mon Sep 17 00:00:00 2001 From: Joel Welling Date: Wed, 26 Aug 2026 15:08:09 -0400 Subject: [PATCH 09/36] croissant now knows about files but not records --- .../build_crate_from_dataset.py | 43 ++++++++++++------- src/crate_builder_script/croissant_wrapper.py | 42 +++++++++++++++--- 2 files changed, 63 insertions(+), 22 deletions(-) diff --git a/src/crate_builder_script/build_crate_from_dataset.py b/src/crate_builder_script/build_crate_from_dataset.py index 309199c..f39c130 100644 --- a/src/crate_builder_script/build_crate_from_dataset.py +++ b/src/crate_builder_script/build_crate_from_dataset.py @@ -154,10 +154,16 @@ def main() -> None: action="store_true", help="Enable debug logging", ) + parser.add_argument( + "--include_all_files", + action="store_true", + help="Include all files, even if some are not data product or qa resources" + ) args = parser.parse_args() target_id = args.target_id outdir = args.outdir debug = args.debug + include_all_files = args.include_all_files if debug: LOGGER.setLevel(logging.DEBUG) @@ -200,33 +206,20 @@ def main() -> None: [crate.add(ent) for ent in ent_l] crate.root_dataset["contributor"] = ent_l - tmpdir = TemporaryDirectory() - - if not os.path.isdir(outdir): - os.makedirs(outdir, exist_ok=True) - wrapped_croissant.write(os.path.join(tmpdir.name, CROISSANT_FILENAME)) - croissant_crate_file = crate.add_file( - os.path.join(tmpdir.name, CROISSANT_FILENAME), - properties={ - "name": "Croissant Metadata Descriptor", - "description": "Machine learning data-loading configurations for this dataset.", - "encodingFormat": "application/ld+json", - "conformsTo": "http://mlcommons.org" - } - ) - # I don't know why this isn't needed #crate.root_dataset.append_to("hasPart", croissant_crate_file) if "files" in ds_info: # This is a derived dataset- include only data products and qa_qc files for fl in ds_info["files"]: - if fl["is_data_product"] or fl["is_qa_qc"]: + if fl["is_data_product"] or fl["is_qa_qc"] or include_all_files: LOGGER.debug(f"Adding {fl['rel_path']}") crate.add_file( api_calls.asset_url(ds_info["uuid"], fl["rel_path"]), validate_url=True ) + wrapped_croissant.add_file(ds_info["uuid"], + fl, blk_idx.get(fl["rel_path"])) else: LOGGER.debug(f"{fl['rel_path']} is not a data product") else: @@ -235,6 +228,24 @@ def main() -> None: api_calls.asset_url(ds_info["uuid"], fl_blk["path"]), validate_url=True ) + # We have no descriptive info for these files, so it's hard + # to see how we could add them to the Croissant object + + tmpdir = TemporaryDirectory() + + if not os.path.isdir(outdir): + os.makedirs(outdir, exist_ok=True) + wrapped_croissant.write(os.path.join(tmpdir.name, CROISSANT_FILENAME)) + croissant_crate_file = crate.add_file( + os.path.join(tmpdir.name, CROISSANT_FILENAME), + properties={ + "name": "Croissant Metadata Descriptor", + "description": "Machine learning data-loading configurations for this dataset.", + "encodingFormat": "application/ld+json", + "conformsTo": "http://mlcommons.org" + } + ) + crate.write_zip(os.path.join(outdir, f"{target_id}_crate.zip")) tmpdir.cleanup() diff --git a/src/crate_builder_script/croissant_wrapper.py b/src/crate_builder_script/croissant_wrapper.py index 100c724..e35a5d8 100644 --- a/src/crate_builder_script/croissant_wrapper.py +++ b/src/crate_builder_script/croissant_wrapper.py @@ -5,12 +5,24 @@ import mlcroissant as mlc -from api_calls import fetch_entity_info, walk_ancestors, listify +from api_calls import fetch_entity_info, walk_ancestors, listify, asset_url LOGGER = logging.getLogger(__name__) HUBMAP = "https://hubmapconsortium.org/" +EDAM_INFO = { + "EDAM_1.24.format_3727" : {"desc":"tiff", "mime":"image/tiff"}, + "EDAM_1.24.format_3464" : {"desc":"json", "mime":"application/json"}, + "EDAM_1.24.format_3508" : {"desc":"pdf"}, + "EDAM_1.24.format_3590" : {"desc":"hdf5", "mime":"application/x-hdf5"}, + "EDAM_1.24.format_3987" : {"desc":"zip", "mime":"application/zip"}, + "EDAM_1.24.format_3752" : {"desc":"csv", "mime":"text/csv"}, + "EDAM_1.24.format_3755" : {"desc":"tsv", "mime":"text/tab-separated-values"}, + "EDAM_1.24.format_3790" : {"desc":"h5ad (anndata)", "mime":"application/x-hdf5"}, + "EDAM_1.24.format_3915" : {"desc":"zarr", "mime":"application/vnd.zarr"}, +} + def _own_dag(entity: dict) -> list: return (entity.get("ingest_metadata") or {}).get("dag_provenance_list", []) @@ -176,11 +188,29 @@ def __init__(self, name: str, description: str): self.file_objects = [] self.record_sets = [] - def add_file(self, file_obj: mlc.FileObject): - self.file_objects.append(file_obj) + def add_file(self, ds_uuid: str, file_info: dict, file_blk: dict | None) -> None: + args = { + "id" : file_info["rel_path"], + "name" : file_info["rel_path"], + "description" : file_info["description"], + "content_url" : asset_url(ds_uuid, file_info["rel_path"]) + } + if file_blk: + args["sha256"] = file_blk["sha256_checksum"] + if edam := file_info.get("edam_term"): + if edam in EDAM_INFO: + args["encoding_formats"] = [EDAM_INFO[edam]["mime"]] + else: + LOGGER.warning(f"Unknown EDAM format {edam} for {pformat(file_info)}") + args["encoding_formats"] = ["application/octet-stream"] + else: + args[encoding_formats] = ["application/octet-stream"] + self.file_objects.append(mlc.FileObject(**args)) + + + # def add_record_set(self, record_set: mlc.RecordSet): + # self.record_sets.append(record_set) - def add_record_set(self, record_set: mlc.RecordSet): - self.record_sets.append(record_set) def write(self, croissant_filename: str): croissant_meta = mlc.Metadata( @@ -189,7 +219,7 @@ def write(self, croissant_filename: str): description=self.description, distribution=self.file_objects, record_sets=self.record_sets, - ctx=mlc.Context( + ctx=mlc.Context( is_live_dataset=False ) ).to_json() From eb6052f2e0b6bf28d49e5819ed1fa92a4d979dca Mon Sep 17 00:00:00 2001 From: Joel Welling Date: Thu, 27 Aug 2026 12:18:25 -0400 Subject: [PATCH 10/36] support version, cite_as, license, date_published for croissant --- .../build_crate_from_dataset.py | 37 +++++++++++++++++-- src/crate_builder_script/croissant_wrapper.py | 31 +++++++++++----- 2 files changed, 54 insertions(+), 14 deletions(-) diff --git a/src/crate_builder_script/build_crate_from_dataset.py b/src/crate_builder_script/build_crate_from_dataset.py index f39c130..77d2e8c 100644 --- a/src/crate_builder_script/build_crate_from_dataset.py +++ b/src/crate_builder_script/build_crate_from_dataset.py @@ -29,8 +29,18 @@ ORCID_URI = "https://orcid.org" OBOLIB_URI = "http://purl.obolibrary.org/obo" -# TARGET_ID = "HBM567.VCBK.562" -# TARGET_ID = "HBM487.HJZB.546" # primary dataset +############### +# Notes- +# - count_versions() is essentially untested, for lack of an example +# - croissant cite_as uses the DOI, and that only gets set for primary datasets. Do we +# want to reference the primary dataset's DOI as the derived dataset's cite_as? +# - unpublished examples: +# TARGET_ID = "HBM567.VCBK.562" +# TARGET_ID = "HBM487.HJZB.546" # primary dataset +# - published examples: +# TARGET_ID = "HBM866.VMBK.952" +# TARGET_ID = "HBM473.QLDT.264" +############### def build_funder_entity(crate: ROCrate) -> ContextEntity: @@ -134,6 +144,14 @@ def build_contributors(crate: ROCrate, contributors: List[dict]) -> List[Context return ent_l +def count_versions(ds_info: dict) -> int: + if "previous_version_uuid" in ds_info: + return (count_versions(api_calls.fetch_entity_info(ds_info["previous_version_uuid"])) + + 1) + else: + return 1 + + def main() -> None: parser = argparse.ArgumentParser() parser.add_argument( @@ -190,16 +208,27 @@ def main() -> None: doi_url = ds_info["doi_url"] crate.root_dataset["identifier"] = doi_url crate.root_dataset["sameAs"] = doi_url + wrapped_croissant.cite_as = doi_url if "published_timestamp" in ds_info: - crate.root_dataset["datePublished"] = str( + date_published = str( datetime.fromtimestamp(ds_info["published_timestamp"] // 1000).astimezone( timezone.utc ) ) + crate.root_dataset["datePublished"] = date_published + wrapped_croissant.date_published = date_published + + license_entity = build_license_entity(crate) + crate.root_dataset["license"] = crate.add(license_entity) + wrapped_croissant.license = license_entity.properties()["url"] - crate.root_dataset["license"] = crate.add(build_license_entity(crate)) crate.root_dataset["funder"] = crate.add(build_funder_entity(crate)) + + ds_version = count_versions(ds_info) + crate.root_dataset["version"] = ds_version + wrapped_croissant.version = ds_version + if contributors := ds_info.get("contributors"): crate.add(build_pi_entity(crate)) ent_l = build_contributors(crate, contributors) diff --git a/src/crate_builder_script/croissant_wrapper.py b/src/crate_builder_script/croissant_wrapper.py index e35a5d8..54e2387 100644 --- a/src/crate_builder_script/croissant_wrapper.py +++ b/src/crate_builder_script/croissant_wrapper.py @@ -187,6 +187,10 @@ def __init__(self, name: str, description: str): self.description = description self.file_objects = [] self.record_sets = [] + self.cite_as = None + self.date_published = None + self.license = None + self.version = None def add_file(self, ds_uuid: str, file_info: dict, file_blk: dict | None) -> None: args = { @@ -213,16 +217,23 @@ def add_file(self, ds_uuid: str, file_info: dict, file_blk: dict | None) -> None def write(self, croissant_filename: str): - croissant_meta = mlc.Metadata( - id="croissant-spec", - name=self.name, - description=self.description, - distribution=self.file_objects, - record_sets=self.record_sets, - ctx=mlc.Context( - is_live_dataset=False - ) - ).to_json() + args = { + "id" : "croissant-spec", + "name" : self.name, + "description" : self.description, + "distribution" : self.file_objects, + "record_sets" : self.record_sets, + "ctx" : mlc.Context(is_live_dataset=False) + } + if self.date_published: + args["date_published"] = self.date_published + if self.license: + args["license"] = self.license + if self.version: + args["version"] = self.version + if self.cite_as: + args["cite_as"] = self.cite_as + croissant_meta = mlc.Metadata(**args).to_json() croissant_meta["@context"].update({ "hubmap": "https://hubmapconsortium.org/", "mlc": "https://mlcommons.org/", From 6d18f779eeeeefd0c1059b70ab348cf047987f61 Mon Sep 17 00:00:00 2001 From: Joel Welling Date: Thu, 27 Aug 2026 16:09:29 -0400 Subject: [PATCH 11/36] previous_version_uuid should be previous_revision_uuid --- .../build_crate_from_dataset.py | 20 +++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/src/crate_builder_script/build_crate_from_dataset.py b/src/crate_builder_script/build_crate_from_dataset.py index 77d2e8c..ce8daba 100644 --- a/src/crate_builder_script/build_crate_from_dataset.py +++ b/src/crate_builder_script/build_crate_from_dataset.py @@ -34,6 +34,12 @@ # - count_versions() is essentially untested, for lack of an example # - croissant cite_as uses the DOI, and that only gets set for primary datasets. Do we # want to reference the primary dataset's DOI as the derived dataset's cite_as? +# - Some EDAM codes, e.g. 3916 (adjacency matrix), are not sufficient to specify +# the mime type to associate with a file. Should we use extensions instead? Some +# specialization would be lost. +# - Writing a croissant for a file in an unpublished dataset results in an error at +# validation time because the file block information from uuid-api has not yet been +# set so the sha256 code is not known. # - unpublished examples: # TARGET_ID = "HBM567.VCBK.562" # TARGET_ID = "HBM487.HJZB.546" # primary dataset @@ -145,9 +151,11 @@ def build_contributors(crate: ROCrate, contributors: List[dict]) -> List[Context def count_versions(ds_info: dict) -> int: - if "previous_version_uuid" in ds_info: - return (count_versions(api_calls.fetch_entity_info(ds_info["previous_version_uuid"])) - + 1) + if "previous_revision_uuid" in ds_info: + return ( + count_versions(api_calls.fetch_entity_info(ds_info["previous_revision_uuid"])) + + 1 + ) else: return 1 @@ -226,6 +234,7 @@ def main() -> None: crate.root_dataset["funder"] = crate.add(build_funder_entity(crate)) ds_version = count_versions(ds_info) + print(f"VERSION: {ds_version}") crate.root_dataset["version"] = ds_version wrapped_croissant.version = ds_version @@ -235,9 +244,6 @@ def main() -> None: [crate.add(ent) for ent in ent_l] crate.root_dataset["contributor"] = ent_l - # I don't know why this isn't needed - #crate.root_dataset.append_to("hasPart", croissant_crate_file) - if "files" in ds_info: # This is a derived dataset- include only data products and qa_qc files for fl in ds_info["files"]: @@ -274,6 +280,8 @@ def main() -> None: "conformsTo": "http://mlcommons.org" } ) + # I don't know why this isn't needed + #crate.root_dataset.append_to("hasPart", croissant_crate_file) crate.write_zip(os.path.join(outdir, f"{target_id}_crate.zip")) tmpdir.cleanup() From 35b26433187d4eece03eb35be60018be5c911219 Mon Sep 17 00:00:00 2001 From: Joel Welling Date: Thu, 27 Aug 2026 16:10:14 -0400 Subject: [PATCH 12/36] add some edam codes; fix edam logic --- src/crate_builder_script/croissant_wrapper.py | 21 +++++++++++-------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/src/crate_builder_script/croissant_wrapper.py b/src/crate_builder_script/croissant_wrapper.py index 54e2387..8f5f822 100644 --- a/src/crate_builder_script/croissant_wrapper.py +++ b/src/crate_builder_script/croissant_wrapper.py @@ -14,13 +14,16 @@ EDAM_INFO = { "EDAM_1.24.format_3727" : {"desc":"tiff", "mime":"image/tiff"}, "EDAM_1.24.format_3464" : {"desc":"json", "mime":"application/json"}, - "EDAM_1.24.format_3508" : {"desc":"pdf"}, + "EDAM_1.24.format_3508" : {"desc":"pdf", "mime":"application/pdf"}, "EDAM_1.24.format_3590" : {"desc":"hdf5", "mime":"application/x-hdf5"}, "EDAM_1.24.format_3987" : {"desc":"zip", "mime":"application/zip"}, "EDAM_1.24.format_3752" : {"desc":"csv", "mime":"text/csv"}, "EDAM_1.24.format_3755" : {"desc":"tsv", "mime":"text/tab-separated-values"}, "EDAM_1.24.format_3790" : {"desc":"h5ad (anndata)", "mime":"application/x-hdf5"}, "EDAM_1.24.format_3915" : {"desc":"zarr", "mime":"application/vnd.zarr"}, + "EDAM_1.24.format_4006" : {"desc":"zarr (spatialdata)", "mime":"application/vnd.zarr"}, + "EDAM_1.24.data_3671" : {"desc":"plain text", "mime":"text/plain"}, + "EDAM_1.24.format_3916" : {"desc":"adjacency matrix", "mime":"text/plain"} } @@ -199,16 +202,16 @@ def add_file(self, ds_uuid: str, file_info: dict, file_blk: dict | None) -> None "description" : file_info["description"], "content_url" : asset_url(ds_uuid, file_info["rel_path"]) } + if edam := file_info.get("edam_term"): + if edam in EDAM_INFO: + args["encoding_formats"] = [EDAM_INFO[edam]["mime"]] + else: + LOGGER.warning(f"Unknown EDAM format {edam} for {pformat(file_info)}") + args["encoding_formats"] = ["application/octet-stream"] + else: + args[encoding_formats] = ["application/octet-stream"] if file_blk: args["sha256"] = file_blk["sha256_checksum"] - if edam := file_info.get("edam_term"): - if edam in EDAM_INFO: - args["encoding_formats"] = [EDAM_INFO[edam]["mime"]] - else: - LOGGER.warning(f"Unknown EDAM format {edam} for {pformat(file_info)}") - args["encoding_formats"] = ["application/octet-stream"] - else: - args[encoding_formats] = ["application/octet-stream"] self.file_objects.append(mlc.FileObject(**args)) From e4612a38538cfa49a4973c918e72241b38646019 Mon Sep 17 00:00:00 2001 From: Joel Welling Date: Fri, 28 Aug 2026 00:38:33 -0400 Subject: [PATCH 13/36] first pass rocrate prov for derived dataset. WIP. --- src/crate_builder_script/api_calls.py | 6 ++ .../build_crate_from_dataset.py | 63 ++++++++++++++++++- src/crate_builder_script/croissant_wrapper.py | 13 ++-- 3 files changed, 75 insertions(+), 7 deletions(-) diff --git a/src/crate_builder_script/api_calls.py b/src/crate_builder_script/api_calls.py index 8ca3d02..4b99e2a 100644 --- a/src/crate_builder_script/api_calls.py +++ b/src/crate_builder_script/api_calls.py @@ -117,3 +117,9 @@ def listify(ancestor_chain: list, omit_test: Callable[[dict], bool]) -> list: for anc in ancestors: rslt.extend(listify([anc], omit_test)) return rslt + + +def is_processed(entity: dict) -> bool: + """Raw vs processed: `creation_action` is 'Create Dataset Activity' vs 'Central Process'.""" + return "process" in (entity.get("creation_action") or "").lower() + diff --git a/src/crate_builder_script/build_crate_from_dataset.py b/src/crate_builder_script/build_crate_from_dataset.py index ce8daba..1d835b8 100644 --- a/src/crate_builder_script/build_crate_from_dataset.py +++ b/src/crate_builder_script/build_crate_from_dataset.py @@ -10,6 +10,8 @@ from rocrate.model.contextentity import ContextEntity from rocrate.model.person import Person +from rocrate.model.dataset import Dataset +from rocrate.model.computerlanguage import ComputerLanguage from rocrate.rocrate import ROCrate import api_calls @@ -45,7 +47,7 @@ # TARGET_ID = "HBM487.HJZB.546" # primary dataset # - published examples: # TARGET_ID = "HBM866.VMBK.952" -# TARGET_ID = "HBM473.QLDT.264" +# TARGET_ID = "HBM473.QLDT.264" # primary dataset ############### @@ -160,6 +162,61 @@ def count_versions(ds_info: dict) -> int: return 1 +def build_derived_prov(ds_info: dict, crate: ROCrate) -> ContextEntity: + prov_chain = api_calls.walk_ancestors( + ds_info, + lambda d: d["entity_type"] == "Dataset" + ) + assert len(prov_chain) == 1 + assert len(prov_chain[0]) == 3 + hubmap_id, parent_chain = prov_chain[0][0], prov_chain[0][2] + parent_id_list = [] + for tuple in parent_chain: + parent_id, parent_info = tuple[0:2] + crate.add(Dataset( + crate, + parent_info["doi_url"], + properties={ + "name": parent_id, + "description": parent_info["description"] + } + )) + parent_id_list.append(parent_info["doi_url"]) + agent = crate.add(Person( # TODO this should be Hubmap internal process? + crate, + "https://orcid.org", + properties={"name": "Joe Schmoe"} + )) + python_lang = crate.add(ComputerLanguage( # TODO this is surely wrong here + crate, + identifier="https://python.org", + properties={"name": "Python", "version": "3.11", "url": "https://python.org"} + )) + workflow_file = crate.add_file( + "https://github.com/hubmapconsortium/data-containers/blob/85441770eafe7da487b35d76112ec099d7b5b8f7/src/crate_builder_script/build_crate_from_dataset.py", + properties={ + "@type": ["File", "SoftwareSourceCode"], + "name":"build_crate_from_dataset.py", + "description": "this should be the dag description. But how to reference CWLs?", + "programmingLanguage": {"@id": python_lang.id} + } + ) + props = { + "@id": "#some_workflow", + "@type": "CreateAction", + "name": "the-create-action", + "startTime": datetime.now().isoformat(), + "endTime": datetime.now().isoformat(), + "agent": {"@id": agent.id}, + "instrument": {"@id": workflow_file.id}, + "object": [{"@id": this_id} for this_id in parent_id_list], + "result": [{"@id": "./"}], # the target dataset + "actionStatus": "CompletedActionStatus" + } + return ContextEntity(crate, identifier=props["@id"], properties=props) + + + def main() -> None: parser = argparse.ArgumentParser() parser.add_argument( @@ -244,6 +301,10 @@ def main() -> None: [crate.add(ent) for ent in ent_l] crate.root_dataset["contributor"] = ent_l + if api_calls.is_processed(ds_info): + print("PING!") + crate.add(build_derived_prov(ds_info, crate)) + if "files" in ds_info: # This is a derived dataset- include only data products and qa_qc files for fl in ds_info["files"]: diff --git a/src/crate_builder_script/croissant_wrapper.py b/src/crate_builder_script/croissant_wrapper.py index 8f5f822..a340876 100644 --- a/src/crate_builder_script/croissant_wrapper.py +++ b/src/crate_builder_script/croissant_wrapper.py @@ -5,7 +5,13 @@ import mlcroissant as mlc -from api_calls import fetch_entity_info, walk_ancestors, listify, asset_url +from api_calls import ( + fetch_entity_info, + walk_ancestors, + listify, + asset_url, + is_processed +) LOGGER = logging.getLogger(__name__) @@ -31,11 +37,6 @@ def _own_dag(entity: dict) -> list: return (entity.get("ingest_metadata") or {}).get("dag_provenance_list", []) -def is_processed(entity: dict) -> bool: - """Raw vs processed: `creation_action` is 'Create Dataset Activity' vs 'Central Process'.""" - return "process" in (entity.get("creation_action") or "").lower() - - def _protocol_dois(md: dict) -> list[str]: dois = [] for k in ("preparation_protocol_doi", "reagent_prep_protocols_io_doi", From cda785419a45c4f3aa33a50470559c65e2f34b48 Mon Sep 17 00:00:00 2001 From: Joel Welling Date: Fri, 28 Aug 2026 10:57:03 -0400 Subject: [PATCH 14/36] fix derived dataset workflow agent --- src/crate_builder_script/api_calls.py | 3 ++ .../build_crate_from_dataset.py | 37 ++++++++++++++----- src/crate_builder_script/croissant_wrapper.py | 5 +-- 3 files changed, 33 insertions(+), 12 deletions(-) diff --git a/src/crate_builder_script/api_calls.py b/src/crate_builder_script/api_calls.py index 4b99e2a..15d18d1 100644 --- a/src/crate_builder_script/api_calls.py +++ b/src/crate_builder_script/api_calls.py @@ -14,6 +14,9 @@ AUTH_TOK = os.environ["AUTH_TOK"] +HUBMAP = "https://hubmapconsortium.org/" + +HUBMAP_ORG_ENTITY = HUBMAP # for lack of a better choice def fetch_entity_info(target_id: str) -> dict[str, Any]: resp = requests.get( diff --git a/src/crate_builder_script/build_crate_from_dataset.py b/src/crate_builder_script/build_crate_from_dataset.py index 1d835b8..43d292b 100644 --- a/src/crate_builder_script/build_crate_from_dataset.py +++ b/src/crate_builder_script/build_crate_from_dataset.py @@ -8,10 +8,17 @@ from typing import Any, List from tempfile import TemporaryDirectory -from rocrate.model.contextentity import ContextEntity -from rocrate.model.person import Person -from rocrate.model.dataset import Dataset -from rocrate.model.computerlanguage import ComputerLanguage +# from rocrate.model.contextentity import ContextEntity +# from rocrate.model.person import Person +# from rocrate.model.dataset import Dataset +# from rocrate.model.computerlanguage import ComputerLanguage +from rocrate.model import ( + ContextEntity, + Person, + Dataset, + ComputerLanguage, + SoftwareApplication +) from rocrate.rocrate import ROCrate import api_calls @@ -182,10 +189,24 @@ def build_derived_prov(ds_info: dict, crate: ROCrate) -> ContextEntity: } )) parent_id_list.append(parent_info["doi_url"]) - agent = crate.add(Person( # TODO this should be Hubmap internal process? + # agent = crate.add(Person( # TODO this should be Hubmap internal process? + # crate, + # "https://orcid.org", + # properties={"name": "Joe Schmoe"} + # )) + hubmap_org = crate.add(ContextEntity( + crate, + api_calls.HUBMAP_ORG_ENTITY, + properties={ + "@type": "Organization", + "name": "HuBMAP Consortium", + "url": api_calls.HUBMAP_ORG_ENTITY + } + )) + agent = crate.add(SoftwareApplication( crate, - "https://orcid.org", - properties={"name": "Joe Schmoe"} + "HuBMAP Process", + properties={"parentOrganization": {"@id": hubmap_org.id}} )) python_lang = crate.add(ComputerLanguage( # TODO this is surely wrong here crate, @@ -291,7 +312,6 @@ def main() -> None: crate.root_dataset["funder"] = crate.add(build_funder_entity(crate)) ds_version = count_versions(ds_info) - print(f"VERSION: {ds_version}") crate.root_dataset["version"] = ds_version wrapped_croissant.version = ds_version @@ -302,7 +322,6 @@ def main() -> None: crate.root_dataset["contributor"] = ent_l if api_calls.is_processed(ds_info): - print("PING!") crate.add(build_derived_prov(ds_info, crate)) if "files" in ds_info: diff --git a/src/crate_builder_script/croissant_wrapper.py b/src/crate_builder_script/croissant_wrapper.py index a340876..25ba99a 100644 --- a/src/crate_builder_script/croissant_wrapper.py +++ b/src/crate_builder_script/croissant_wrapper.py @@ -10,13 +10,12 @@ walk_ancestors, listify, asset_url, - is_processed + is_processed, + HUBMAP ) LOGGER = logging.getLogger(__name__) -HUBMAP = "https://hubmapconsortium.org/" - EDAM_INFO = { "EDAM_1.24.format_3727" : {"desc":"tiff", "mime":"image/tiff"}, "EDAM_1.24.format_3464" : {"desc":"json", "mime":"application/json"}, From 932e00d24cd1c5ce77e575f6fd7a829e571cf531 Mon Sep 17 00:00:00 2001 From: Joel Welling Date: Mon, 31 Aug 2026 16:20:28 -0400 Subject: [PATCH 15/36] workflow crate passes validation --- .../build_crate_from_dataset.py | 89 +++++++++++++++++-- 1 file changed, 81 insertions(+), 8 deletions(-) diff --git a/src/crate_builder_script/build_crate_from_dataset.py b/src/crate_builder_script/build_crate_from_dataset.py index 43d292b..6e31f23 100644 --- a/src/crate_builder_script/build_crate_from_dataset.py +++ b/src/crate_builder_script/build_crate_from_dataset.py @@ -189,11 +189,6 @@ def build_derived_prov(ds_info: dict, crate: ROCrate) -> ContextEntity: } )) parent_id_list.append(parent_info["doi_url"]) - # agent = crate.add(Person( # TODO this should be Hubmap internal process? - # crate, - # "https://orcid.org", - # properties={"name": "Joe Schmoe"} - # )) hubmap_org = crate.add(ContextEntity( crate, api_calls.HUBMAP_ORG_ENTITY, @@ -216,12 +211,13 @@ def build_derived_prov(ds_info: dict, crate: ROCrate) -> ContextEntity: workflow_file = crate.add_file( "https://github.com/hubmapconsortium/data-containers/blob/85441770eafe7da487b35d76112ec099d7b5b8f7/src/crate_builder_script/build_crate_from_dataset.py", properties={ - "@type": ["File", "SoftwareSourceCode"], + "@type": ["File", "SoftwareSourceCode", "ComputationalWorkflow"], "name":"build_crate_from_dataset.py", "description": "this should be the dag description. But how to reference CWLs?", "programmingLanguage": {"@id": python_lang.id} } ) + crate.mainEntity = workflow_file props = { "@id": "#some_workflow", "@type": "CreateAction", @@ -285,7 +281,74 @@ def main() -> None: # The dataset DOIs point to the Portal, which is basically a landing page, which is # forbidden as the direct link for a dataset under FAIR. So we can't use the DOI # as the crate root dataset id. - crate = ROCrate() + crate = ROCrate(version="1.1") + print(f"CRATE: \n{pformat(crate.root_dataset.get('conformsTo'))}") + + base_crate_ctx_id = f"https://w3id.org/ro/crate/{crate.version}" + # base_crate_ctx_id = f"https://w3id.org" + print(f"CRATE CTX ID: " + base_crate_ctx_id) + crate_profile = crate.add(ContextEntity( + crate, + base_crate_ctx_id, + properties={ + "@type": ["CreativeWork", "Profile"], + "name": "RO-Crate Profile", + "version": crate.version + } + )) + proc_profile = crate.add(ContextEntity( + crate, + "https://w3id.org/ro/wfrun/process/0.5", + properties={ + "@type": ["CreativeWork", "Profile"], + "name": "Process Run Crate Profile", + "version": "0.5" + } + )) + wf_profile = crate.add(ContextEntity( + crate, + "https://w3id.org/ro/wfrun/workflow/0.5", + properties={ + "@type": ["CreativeWork", "Profile"], + "name": "Workflow Run Crate Profile", + "version": "0.5", + # "isProfileOf": {"@id": proc_profile.id} + } + )) + wfc_profile = crate.add(ContextEntity( + crate, + "https://w3id.org/workflowhub/workflow-ro-crate/1.0", + properties={ + "@type": ["CreativeWork", "Profile"], + "name": "Workflow Run RO-Crate", + "version": "1.0" + } + )) + crate.root_dataset.append_to("conformsTo", {"@id": crate_profile.id}) + # crate.root_dataset.append_to("conformsTo", {"@id": "https://w3id.org"}) + crate.root_dataset.append_to("conformsTo", {"@id": proc_profile.id}) + crate.root_dataset.append_to("conformsTo", {"@id": wf_profile.id}) + crate.root_dataset.append_to("conformsTo", {"@id": wfc_profile.id}) + # crate.metadata.extra_contexts.append("https://w3id.org/ro/wfrun/process/0.5") + # crate.metadata.extra_contexts.append("https://w3id.org/ro/wfrun/workflow/0.5") + crate.metadata.extra_contexts.append("https://w3id.org/ro/terms/workflow-run/context") + print(f"CRATE: \n{pformat(crate.metadata['conformsTo'])}") + # crate.metadata["conformsTo"] = [ + # # crate.metadata["conformsTo"], + # {"@id": crate_profile.id}, + # {"@id": proc_profile.id}, + # {"@id": wf_profile.id}, + # {"@id": wfc_profile.id} + # ] + # crate.metadata["conformsTo"] = {"@id": wfc_profile.id} + # crate.metadata["conformsTo"] = [ + # crate_profile.id, + # proc_profile.id, + # wf_profile.id, + # wfc_profile.id + # ] + crate.metadata["conformsTo"] = {"@id": crate_profile.id} + crate.root_dataset["name"] = target_id crate.root_dataset["description"] = ds_info["title"] wrapped_croissant = CroissantWrapper(target_id, ds_info["title"]) @@ -351,13 +414,23 @@ def main() -> None: if not os.path.isdir(outdir): os.makedirs(outdir, exist_ok=True) wrapped_croissant.write(os.path.join(tmpdir.name, CROISSANT_FILENAME)) + crate.add(ContextEntity( + crate, + "http://mlcommons.org/croissant/1.0", + properties={ + "@type": ["CreativeWork", "Profile"], + "name": "MLCommons Croissant Format Specification", + "version": "1.0", + "url": "https://docs.mlcommons.org/croissant/docs/crossant-spec-1.0.html" + } + )) croissant_crate_file = crate.add_file( os.path.join(tmpdir.name, CROISSANT_FILENAME), properties={ "name": "Croissant Metadata Descriptor", "description": "Machine learning data-loading configurations for this dataset.", "encodingFormat": "application/ld+json", - "conformsTo": "http://mlcommons.org" + "conformsTo": {"@id": "http://mlcommons.org/croissant/1.0"} } ) # I don't know why this isn't needed From ab54f5a15f35dff8dcfd734c6acaf5ed73271548 Mon Sep 17 00:00:00 2001 From: Joel Welling Date: Mon, 31 Aug 2026 16:41:31 -0400 Subject: [PATCH 16/36] cleanup --- .../build_crate_from_dataset.py | 122 ++++++++---------- 1 file changed, 53 insertions(+), 69 deletions(-) diff --git a/src/crate_builder_script/build_crate_from_dataset.py b/src/crate_builder_script/build_crate_from_dataset.py index 6e31f23..92ae910 100644 --- a/src/crate_builder_script/build_crate_from_dataset.py +++ b/src/crate_builder_script/build_crate_from_dataset.py @@ -232,7 +232,51 @@ def build_derived_prov(ds_info: dict, crate: ROCrate) -> ContextEntity: } return ContextEntity(crate, identifier=props["@id"], properties=props) - + +def build_profiles(crate: ROCrate) -> tuple: + """ + Build ContextElements for several profiles needed to describe a workflow. + """ + base_crate_ctx_id = f"https://w3id.org/ro/crate/{crate.version}" + crate_profile = crate.add(ContextEntity( + crate, + base_crate_ctx_id, + properties={ + "@type": ["CreativeWork", "Profile"], + "name": "RO-Crate Profile", + "version": crate.version + } + )) + proc_profile = crate.add(ContextEntity( + crate, + "https://w3id.org/ro/wfrun/process/0.5", + properties={ + "@type": ["CreativeWork", "Profile"], + "name": "Process Run Crate Profile", + "version": "0.5" + } + )) + wf_profile = crate.add(ContextEntity( + crate, + "https://w3id.org/ro/wfrun/workflow/0.5", + properties={ + "@type": ["CreativeWork", "Profile"], + "name": "Workflow Run Crate Profile", + "version": "0.5", + # "isProfileOf": {"@id": proc_profile.id} + } + )) + wfc_profile = crate.add(ContextEntity( + crate, + "https://w3id.org/workflowhub/workflow-ro-crate/1.0", + properties={ + "@type": ["CreativeWork", "Profile"], + "name": "Workflow Run RO-Crate", + "version": "1.0" + } + )) + return (crate_profile, proc_profile, wf_profile, wfc_profile) + def main() -> None: parser = argparse.ArgumentParser() @@ -281,78 +325,20 @@ def main() -> None: # The dataset DOIs point to the Portal, which is basically a landing page, which is # forbidden as the direct link for a dataset under FAIR. So we can't use the DOI # as the crate root dataset id. + # The version needs to be 1.1 to avoid compatibility problems with the workflow + # profile that seem to exist for crate profile 1.2. crate = ROCrate(version="1.1") - print(f"CRATE: \n{pformat(crate.root_dataset.get('conformsTo'))}") - base_crate_ctx_id = f"https://w3id.org/ro/crate/{crate.version}" - # base_crate_ctx_id = f"https://w3id.org" - print(f"CRATE CTX ID: " + base_crate_ctx_id) - crate_profile = crate.add(ContextEntity( - crate, - base_crate_ctx_id, - properties={ - "@type": ["CreativeWork", "Profile"], - "name": "RO-Crate Profile", - "version": crate.version - } - )) - proc_profile = crate.add(ContextEntity( - crate, - "https://w3id.org/ro/wfrun/process/0.5", - properties={ - "@type": ["CreativeWork", "Profile"], - "name": "Process Run Crate Profile", - "version": "0.5" - } - )) - wf_profile = crate.add(ContextEntity( - crate, - "https://w3id.org/ro/wfrun/workflow/0.5", - properties={ - "@type": ["CreativeWork", "Profile"], - "name": "Workflow Run Crate Profile", - "version": "0.5", - # "isProfileOf": {"@id": proc_profile.id} - } - )) - wfc_profile = crate.add(ContextEntity( - crate, - "https://w3id.org/workflowhub/workflow-ro-crate/1.0", - properties={ - "@type": ["CreativeWork", "Profile"], - "name": "Workflow Run RO-Crate", - "version": "1.0" - } - )) - crate.root_dataset.append_to("conformsTo", {"@id": crate_profile.id}) - # crate.root_dataset.append_to("conformsTo", {"@id": "https://w3id.org"}) - crate.root_dataset.append_to("conformsTo", {"@id": proc_profile.id}) - crate.root_dataset.append_to("conformsTo", {"@id": wf_profile.id}) - crate.root_dataset.append_to("conformsTo", {"@id": wfc_profile.id}) - # crate.metadata.extra_contexts.append("https://w3id.org/ro/wfrun/process/0.5") - # crate.metadata.extra_contexts.append("https://w3id.org/ro/wfrun/workflow/0.5") + (crate_profile, proc_profile, wf_profile, wfc_profile) = build_profiles(crate) + for profile in [crate_profile, proc_profile, wf_profile, wfc_profile]: + crate.root_dataset.append_to("conformsTo", {"@id": profile.id}) + crate.metadata.extra_contexts.append("https://w3id.org/ro/terms/workflow-run/context") - print(f"CRATE: \n{pformat(crate.metadata['conformsTo'])}") - # crate.metadata["conformsTo"] = [ - # # crate.metadata["conformsTo"], - # {"@id": crate_profile.id}, - # {"@id": proc_profile.id}, - # {"@id": wf_profile.id}, - # {"@id": wfc_profile.id} - # ] - # crate.metadata["conformsTo"] = {"@id": wfc_profile.id} - # crate.metadata["conformsTo"] = [ - # crate_profile.id, - # proc_profile.id, - # wf_profile.id, - # wfc_profile.id - # ] - crate.metadata["conformsTo"] = {"@id": crate_profile.id} - crate.root_dataset["name"] = target_id - crate.root_dataset["description"] = ds_info["title"] wrapped_croissant = CroissantWrapper(target_id, ds_info["title"]) + crate.root_dataset["name"] = target_id + crate.root_dataset["description"] = ds_info["title"] if "doi_url" in ds_info: doi_url = ds_info["doi_url"] crate.root_dataset["identifier"] = doi_url @@ -433,8 +419,6 @@ def main() -> None: "conformsTo": {"@id": "http://mlcommons.org/croissant/1.0"} } ) - # I don't know why this isn't needed - #crate.root_dataset.append_to("hasPart", croissant_crate_file) crate.write_zip(os.path.join(outdir, f"{target_id}_crate.zip")) tmpdir.cleanup() From 59397b5de16c1b195db0ec54b96cfe190acc69f9 Mon Sep 17 00:00:00 2001 From: Joel Welling Date: Mon, 31 Aug 2026 16:42:51 -0400 Subject: [PATCH 17/36] more cleanup --- src/crate_builder_script/build_crate_from_dataset.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/crate_builder_script/build_crate_from_dataset.py b/src/crate_builder_script/build_crate_from_dataset.py index 92ae910..914cb5c 100644 --- a/src/crate_builder_script/build_crate_from_dataset.py +++ b/src/crate_builder_script/build_crate_from_dataset.py @@ -329,8 +329,7 @@ def main() -> None: # profile that seem to exist for crate profile 1.2. crate = ROCrate(version="1.1") - (crate_profile, proc_profile, wf_profile, wfc_profile) = build_profiles(crate) - for profile in [crate_profile, proc_profile, wf_profile, wfc_profile]: + for profile in build_profiles(crate): crate.root_dataset.append_to("conformsTo", {"@id": profile.id}) crate.metadata.extra_contexts.append("https://w3id.org/ro/terms/workflow-run/context") From 5cd94bbaef54df9439fa8af4f419553ee09d368f Mon Sep 17 00:00:00 2001 From: Joel Welling Date: Tue, 1 Sep 2026 11:27:49 -0400 Subject: [PATCH 18/36] refactor out extractors --- src/crate_builder_script/api_calls.py | 73 ----------------- .../build_crate_from_dataset.py | 32 ++++---- src/crate_builder_script/croissant_wrapper.py | 14 +--- src/crate_builder_script/extractors.py | 78 +++++++++++++++++++ 4 files changed, 99 insertions(+), 98 deletions(-) create mode 100644 src/crate_builder_script/extractors.py diff --git a/src/crate_builder_script/api_calls.py b/src/crate_builder_script/api_calls.py index 15d18d1..5504850 100644 --- a/src/crate_builder_script/api_calls.py +++ b/src/crate_builder_script/api_calls.py @@ -2,7 +2,6 @@ import os from pprint import pformat from typing import Any -from collections.abc import Callable import requests @@ -54,75 +53,3 @@ def fetch_uuid_files_info(target_id: str) -> dict[str, Any]: def asset_url(uuid: str, rel_path: str) -> str: return f"{ASSETS_API}/{uuid}/{rel_path}" - - -def walk_ancestors( - entity: dict, - continue_test: Callable[[dict], bool] = lambda ent: True - ) -> list[tuple]: - """ - Given an entity dictionary, return a list of tuples. Each tuple has - the form: - (hubmap_id entity_dict list-of-ancestors) - where list-of-ancestors is None or a list of tuples of the same form. - - continue_test takes an entity dict as a parameter and returns True if - the descent should continue to the children of that entity, False otherwise. - """ - rslt = [] - if not continue_test(entity): - return rslt - e_type = entity.get("entity_type") - e_id = entity.get("hubmap_id") - LOGGER.debug(f"walk_ancestors {e_id} {e_type}") - if e_type == "Dataset": - ancs = [walk_ancestors(anc, continue_test) - for anc in entity.get("direct_ancestors", [])] - if ancs: - all_tuples = [] - for sub_list in ancs: - assert isinstance(sub_list, list) - all_tuples.extend(sub_list) - rslt.append((e_id, entity, all_tuples)) - else: - md = entity.get("metadata", {}) - if "parent_sample_id" in md: - # The parent is a sample - samp_id = md["parent_sample_id"] - samp_entity = fetch_entity_info(samp_id) - rslt.append((e_id, entity, walk_ancestors(samp_entity, continue_test))) - elif e_type in ("Sample", "Donor"): - e_cat = entity.get("sample_category", "UNKNOWN SAMPLE CATEGORY") - LOGGER.debug(f"walk_ancestors sample category is {e_cat}") - if "direct_ancestor" not in entity: - LOGGER.debug(f"walk_ancestors fetching dead-end sample {e_id}") - entity = fetch_entity_info(e_id) - LOGGER.debug("walk_ancestors fetch yielded:\n%s", - pformat(entity, depth=2)) - LOGGER.debug("walk_ancestors end of walk jump result") - new_entity = entity.get("direct_ancestor", {}) - if e_type == "Donor": - rslt.append((e_id, entity, None)) - else: - rslt.append((e_id, entity, walk_ancestors(new_entity, continue_test))) - else: - LOGGER.warning(f"walk_ancestors UNKNOWN ETYPE {e_type} for {e_id}") - return rslt - - -def listify(ancestor_chain: list, omit_test: Callable[[dict], bool]) -> list: - assert len(ancestor_chain) == 1, "listify must start on a 1-tuple chain" - hubmap_id, entity_dict, ancestors = ancestor_chain[0] - rslt = [] - if not omit_test(entity_dict): - rslt.append(entity_dict) - if ancestors: - for anc in ancestors: - rslt.extend(listify([anc], omit_test)) - return rslt - - -def is_processed(entity: dict) -> bool: - """Raw vs processed: `creation_action` is 'Create Dataset Activity' vs 'Central Process'.""" - return "process" in (entity.get("creation_action") or "").lower() - diff --git a/src/crate_builder_script/build_crate_from_dataset.py b/src/crate_builder_script/build_crate_from_dataset.py index 914cb5c..63ae39d 100644 --- a/src/crate_builder_script/build_crate_from_dataset.py +++ b/src/crate_builder_script/build_crate_from_dataset.py @@ -5,13 +5,9 @@ from collections import defaultdict from datetime import datetime, timezone from pprint import pformat, pprint -from typing import Any, List +from typing import List from tempfile import TemporaryDirectory -# from rocrate.model.contextentity import ContextEntity -# from rocrate.model.person import Person -# from rocrate.model.dataset import Dataset -# from rocrate.model.computerlanguage import ComputerLanguage from rocrate.model import ( ContextEntity, Person, @@ -21,7 +17,13 @@ ) from rocrate.rocrate import ROCrate -import api_calls +from api_calls import ( + fetch_entity_info, + fetch_uuid_files_info, + asset_url, + HUBMAP_ORG_ENTITY +) +from extractors import walk_ancestors, is_processed from croissant_wrapper import CroissantWrapper logging.basicConfig( @@ -162,7 +164,7 @@ def build_contributors(crate: ROCrate, contributors: List[dict]) -> List[Context def count_versions(ds_info: dict) -> int: if "previous_revision_uuid" in ds_info: return ( - count_versions(api_calls.fetch_entity_info(ds_info["previous_revision_uuid"])) + count_versions(fetch_entity_info(ds_info["previous_revision_uuid"])) + 1 ) else: @@ -170,7 +172,7 @@ def count_versions(ds_info: dict) -> int: def build_derived_prov(ds_info: dict, crate: ROCrate) -> ContextEntity: - prov_chain = api_calls.walk_ancestors( + prov_chain = walk_ancestors( ds_info, lambda d: d["entity_type"] == "Dataset" ) @@ -191,11 +193,11 @@ def build_derived_prov(ds_info: dict, crate: ROCrate) -> ContextEntity: parent_id_list.append(parent_info["doi_url"]) hubmap_org = crate.add(ContextEntity( crate, - api_calls.HUBMAP_ORG_ENTITY, + HUBMAP_ORG_ENTITY, properties={ "@type": "Organization", "name": "HuBMAP Consortium", - "url": api_calls.HUBMAP_ORG_ENTITY + "url": HUBMAP_ORG_ENTITY } )) agent = crate.add(SoftwareApplication( @@ -315,9 +317,9 @@ def main() -> None: logging.getLogger("urllib3").setLevel(logging.DEBUG) logging.getLogger("api_calls").setLevel(logging.DEBUG) logging.getLogger("croissant_wrapper").setLevel(logging.DEBUG) - ds_info = api_calls.fetch_entity_info(target_id) + ds_info = fetch_entity_info(target_id) - uuid_files = api_calls.fetch_uuid_files_info(target_id) + uuid_files = fetch_uuid_files_info(target_id) blk_idx = {} for file_blk in uuid_files: blk_idx[file_blk["path"]] = file_blk @@ -369,7 +371,7 @@ def main() -> None: [crate.add(ent) for ent in ent_l] crate.root_dataset["contributor"] = ent_l - if api_calls.is_processed(ds_info): + if is_processed(ds_info): crate.add(build_derived_prov(ds_info, crate)) if "files" in ds_info: @@ -378,7 +380,7 @@ def main() -> None: if fl["is_data_product"] or fl["is_qa_qc"] or include_all_files: LOGGER.debug(f"Adding {fl['rel_path']}") crate.add_file( - api_calls.asset_url(ds_info["uuid"], fl["rel_path"]), + asset_url(ds_info["uuid"], fl["rel_path"]), validate_url=True ) wrapped_croissant.add_file(ds_info["uuid"], @@ -388,7 +390,7 @@ def main() -> None: else: for fl_blk in blk_idx.values(): crate.add_file( - api_calls.asset_url(ds_info["uuid"], fl_blk["path"]), + asset_url(ds_info["uuid"], fl_blk["path"]), validate_url=True ) # We have no descriptive info for these files, so it's hard diff --git a/src/crate_builder_script/croissant_wrapper.py b/src/crate_builder_script/croissant_wrapper.py index 25ba99a..c347b7f 100644 --- a/src/crate_builder_script/croissant_wrapper.py +++ b/src/crate_builder_script/croissant_wrapper.py @@ -1,18 +1,12 @@ import json import logging -from os import walk -from pprint import pformat, pprint +from pprint import pformat import mlcroissant as mlc -from api_calls import ( - fetch_entity_info, - walk_ancestors, - listify, - asset_url, - is_processed, - HUBMAP -) +from api_calls import fetch_entity_info, asset_url, HUBMAP + +from extractors import walk_ancestors, listify, is_processed LOGGER = logging.getLogger(__name__) diff --git a/src/crate_builder_script/extractors.py b/src/crate_builder_script/extractors.py new file mode 100644 index 0000000..44da32f --- /dev/null +++ b/src/crate_builder_script/extractors.py @@ -0,0 +1,78 @@ +import logging +from pprint import pformat +from collections.abc import Callable + +from api_calls import fetch_entity_info + +LOGGER = logging.getLogger(__name__) + +def walk_ancestors( + entity: dict, + continue_test: Callable[[dict], bool] = lambda ent: True + ) -> list[tuple]: + """ + Given an entity dictionary, return a list of tuples. Each tuple has + the form: + (hubmap_id entity_dict list-of-ancestors) + where list-of-ancestors is None or a list of tuples of the same form. + + continue_test takes an entity dict as a parameter and returns True if + the descent should continue to the children of that entity, False otherwise. + """ + rslt = [] + if not continue_test(entity): + return rslt + e_type = entity.get("entity_type") + e_id = entity.get("hubmap_id") + LOGGER.debug(f"walk_ancestors {e_id} {e_type}") + if e_type == "Dataset": + ancs = [walk_ancestors(anc, continue_test) + for anc in entity.get("direct_ancestors", [])] + if ancs: + all_tuples = [] + for sub_list in ancs: + assert isinstance(sub_list, list) + all_tuples.extend(sub_list) + rslt.append((e_id, entity, all_tuples)) + else: + md = entity.get("metadata", {}) + if "parent_sample_id" in md: + # The parent is a sample + samp_id = md["parent_sample_id"] + samp_entity = fetch_entity_info(samp_id) + rslt.append((e_id, entity, walk_ancestors(samp_entity, continue_test))) + elif e_type in ("Sample", "Donor"): + e_cat = entity.get("sample_category", "UNKNOWN SAMPLE CATEGORY") + LOGGER.debug(f"walk_ancestors sample category is {e_cat}") + if "direct_ancestor" not in entity: + LOGGER.debug(f"walk_ancestors fetching dead-end sample {e_id}") + entity = fetch_entity_info(e_id) + LOGGER.debug("walk_ancestors fetch yielded:\n%s", + pformat(entity, depth=2)) + LOGGER.debug("walk_ancestors end of walk jump result") + new_entity = entity.get("direct_ancestor", {}) + if e_type == "Donor": + rslt.append((e_id, entity, None)) + else: + rslt.append((e_id, entity, walk_ancestors(new_entity, continue_test))) + else: + LOGGER.warning(f"walk_ancestors UNKNOWN ETYPE {e_type} for {e_id}") + return rslt + + +def listify(ancestor_chain: list, omit_test: Callable[[dict], bool]) -> list: + assert len(ancestor_chain) == 1, "listify must start on a 1-tuple chain" + hubmap_id, entity_dict, ancestors = ancestor_chain[0] + rslt = [] + if not omit_test(entity_dict): + rslt.append(entity_dict) + if ancestors: + for anc in ancestors: + rslt.extend(listify([anc], omit_test)) + return rslt + + +def is_processed(entity: dict) -> bool: + """Raw vs processed: `creation_action` is 'Create Dataset Activity' vs 'Central Process'.""" + return "process" in (entity.get("creation_action") or "").lower() + From 950f7226c61c950315f7168604f8c068075d95c1 Mon Sep 17 00:00:00 2001 From: Joel Welling Date: Tue, 1 Sep 2026 11:41:25 -0400 Subject: [PATCH 19/36] move pipeline_steps to extractors --- src/crate_builder_script/croissant_wrapper.py | 26 +++---------------- src/crate_builder_script/extractors.py | 20 ++++++++++++++ 2 files changed, 24 insertions(+), 22 deletions(-) diff --git a/src/crate_builder_script/croissant_wrapper.py b/src/crate_builder_script/croissant_wrapper.py index c347b7f..2ac3926 100644 --- a/src/crate_builder_script/croissant_wrapper.py +++ b/src/crate_builder_script/croissant_wrapper.py @@ -6,7 +6,7 @@ from api_calls import fetch_entity_info, asset_url, HUBMAP -from extractors import walk_ancestors, listify, is_processed +from extractors import walk_ancestors, listify, is_processed, pipeline_steps LOGGER = logging.getLogger(__name__) @@ -26,10 +26,6 @@ } -def _own_dag(entity: dict) -> list: - return (entity.get("ingest_metadata") or {}).get("dag_provenance_list", []) - - def _protocol_dois(md: dict) -> list[str]: dois = [] for k in ("preparation_protocol_doi", "reagent_prep_protocols_io_doi", @@ -41,20 +37,6 @@ def _protocol_dois(md: dict) -> list[str]: return dois -def _pipeline_steps(dag_list: list) -> list[dict]: - steps, seen = [], set() - for s in dag_list or []: - repo = (s.get("origin") or "").strip().replace(".git", "") - name = repo.rsplit("/", 1)[-1] if repo else (s.get("name") or "") - commit = (s.get("hash") or "")[:7] - cwl = s.get("name") or "" - key = (name, commit, cwl) - if not name or key in seen: - continue - seen.add(key) - steps.append({"name": name, "repo": repo, "commit": commit, "cwl": cwl}) - return steps - def _acquisition_activity(entity: dict, md: dict) -> dict: def agent(name, role=None, org=False): n = {"@type": "prov:Organization" if org else "prov:Person", "schema:name": name} @@ -114,9 +96,9 @@ def node(anc): return derived -def _pipeline_activity(dag_list: list) -> dict: +def _pipeline_activity(entity: dict) -> dict: agents = [] - for st in _pipeline_steps(dag_list): + for st in pipeline_steps(entity): a = {"@type": ["prov:SoftwareAgent", "schema:SoftwareApplication"], "schema:name": st["name"] + (f" [{st['cwl']}]" if st["cwl"] else ""), "schema:codeRepository": st["repo"], "hubmap:commit": st["commit"]} @@ -152,7 +134,7 @@ def build_embedded_provenance(entity, md, descendants=None) -> dict: "prov:wasGeneratedBy": _acquisition_activity(raw_entity, raw_md or {}), "prov:wasDerivedFrom": _specimen_chain(raw_entity) } - return {"prov:wasGeneratedBy": _pipeline_activity(_own_dag(entity)), + return {"prov:wasGeneratedBy": _pipeline_activity(entity), "prov:wasDerivedFrom": raw_node} provo = {"prov:wasGeneratedBy": _acquisition_activity(entity, md)} if chain := _specimen_chain(entity): diff --git a/src/crate_builder_script/extractors.py b/src/crate_builder_script/extractors.py index 44da32f..3da07e5 100644 --- a/src/crate_builder_script/extractors.py +++ b/src/crate_builder_script/extractors.py @@ -76,3 +76,23 @@ def is_processed(entity: dict) -> bool: """Raw vs processed: `creation_action` is 'Create Dataset Activity' vs 'Central Process'.""" return "process" in (entity.get("creation_action") or "").lower() + +def _own_dag(entity: dict) -> list: + return (entity.get("ingest_metadata") or {}).get("dag_provenance_list", []) + + +def pipeline_steps(entity: dict) -> list[dict]: + dag_list = _own_dag(entity) + steps, seen = [], set() + for s in dag_list or []: + repo = (s.get("origin") or "").strip().replace(".git", "") + name = repo.rsplit("/", 1)[-1] if repo else (s.get("name") or "") + commit = (s.get("hash") or "")[:7] + cwl = s.get("name") or "" + key = (name, commit, cwl) + if not name or key in seen: + continue + seen.add(key) + steps.append({"name": name, "repo": repo, "commit": commit, "cwl": cwl}) + return steps + From 1b0a5c11c79c5c0ae550b1a9b7bf67c7512c42de Mon Sep 17 00:00:00 2001 From: Joel Welling Date: Tue, 1 Sep 2026 12:44:03 -0400 Subject: [PATCH 20/36] introduce WrappedEntity --- src/crate_builder_script/croissant_wrapper.py | 33 +++++++-------- src/crate_builder_script/extractors.py | 40 +++++++++++++++++++ 2 files changed, 57 insertions(+), 16 deletions(-) diff --git a/src/crate_builder_script/croissant_wrapper.py b/src/crate_builder_script/croissant_wrapper.py index 2ac3926..f5a2284 100644 --- a/src/crate_builder_script/croissant_wrapper.py +++ b/src/crate_builder_script/croissant_wrapper.py @@ -6,7 +6,7 @@ from api_calls import fetch_entity_info, asset_url, HUBMAP -from extractors import walk_ancestors, listify, is_processed, pipeline_steps +from extractors import WrappedEntity, walk_ancestors, listify, is_processed, pipeline_steps LOGGER = logging.getLogger(__name__) @@ -37,7 +37,7 @@ def _protocol_dois(md: dict) -> list[str]: return dois -def _acquisition_activity(entity: dict, md: dict) -> dict: +def _acquisition_activity(entity: WrappedEntity, md: dict) -> dict: def agent(name, role=None, org=False): n = {"@type": "prov:Organization" if org else "prov:Person", "schema:name": name} if role: @@ -69,7 +69,7 @@ def agent(name, role=None, org=False): return {k: v for k, v in act.items() if v not in (None, "", [])} -def _specimen_chain(entity: dict, recur=0) -> dict: +def _specimen_chain(entity: WrappedEntity) -> dict: def node(anc): n = {"@type": "prov:Entity", "@id": HUBMAP + (anc.get("hubmap_id") or ""), "schema:name": anc.get("hubmap_id"), "hubmap:entityType": anc.get("entity_type"), @@ -81,8 +81,7 @@ def node(anc): n["hubmap:dimensions"] = {"x": r.get("x_dimension"), "y": r.get("y_dimension"), "z": r.get("z_dimension"), "unit": r.get("dimension_units")} return {k: v for k, v in n.items() if v not in (None, "", [])} - ancs = listify( - walk_ancestors(entity), + ancs = entity.list_ancestors( omit_test=lambda dct: dct["entity_type"]=="Dataset" ) order = {"section": 0, "block": 1, "organ": 2} @@ -96,9 +95,9 @@ def node(anc): return derived -def _pipeline_activity(entity: dict) -> dict: +def _pipeline_activity(entity: WrappedEntity) -> dict: agents = [] - for st in pipeline_steps(entity): + for st in entity.pipeline_steps(): a = {"@type": ["prov:SoftwareAgent", "schema:SoftwareApplication"], "schema:name": st["name"] + (f" [{st['cwl']}]" if st["cwl"] else ""), "schema:codeRepository": st["repo"], "hubmap:commit": st["commit"]} @@ -109,23 +108,25 @@ def _pipeline_activity(entity: dict) -> dict: return act -def build_embedded_provenance(entity, md, descendants=None) -> dict: +def build_embedded_provenance( + entity: WrappedEntity, + md: dict | None, + descendants: list | None =None) -> dict: """ PROCESSED subject: wasGeneratedBy its own pipeline; wasDerivedFrom the raw parent (which carries the acquisition activity + specimen chain). RAW subject: wasGeneratedBy acquisition; wasDerivedFrom specimen chain; + a light forward pointer to processed versions. """ - if is_processed(entity): + if entity.is_processed: hubmap_id = entity["hubmap_id"] - ancestor_chain = walk_ancestors( - entity, + ancestor_chain = entity.walk_ancestors( lambda d: d["entity_type"] == "Dataset" ) assert len(ancestor_chain) == 1, "internal error walking ancestors" check_id, ignored_entity, ancestors = ancestor_chain[0] assert check_id == hubmap_id assert len(ancestors) == 1, f"Dataset {hubmap_id} has too many ancestors" - raw_id, raw_entity, ignored = ancestors[0] + raw_entity = WrappedEntity(ancestors[0][1]) raw_md = raw_entity.get("metadata") raw_node = { "@type": "prov:Entity", "@id": HUBMAP + (raw_entity.get("hubmap_id") or ""), @@ -219,12 +220,12 @@ def write(self, croissant_filename: str): "prov": "http://www.w3.org/ns/prov#", "schema": "http://schema.org/" }) - entity_dict = fetch_entity_info(self.name) + entity = WrappedEntity(fetch_entity_info(self.name)) croissant_meta.update( build_embedded_provenance( - entity_dict, - md=entity_dict.get("metadata"), # md - descendants=entity_dict.get("direct_descendants", []) + entity, + md=entity.get("metadata"), # md + descendants=entity.get("direct_descendants", []) ) ) with open(croissant_filename, "w", encoding="utf-8") as f: diff --git a/src/crate_builder_script/extractors.py b/src/crate_builder_script/extractors.py index 3da07e5..ab92df0 100644 --- a/src/crate_builder_script/extractors.py +++ b/src/crate_builder_script/extractors.py @@ -1,6 +1,7 @@ import logging from pprint import pformat from collections.abc import Callable +from typing import Any from api_calls import fetch_entity_info @@ -96,3 +97,42 @@ def pipeline_steps(entity: dict) -> list[dict]: steps.append({"name": name, "repo": repo, "commit": commit, "cwl": cwl}) return steps + +class WrappedEntity: + def __init__(self, entity: dict): + self._entity = entity + + def get(self, key: Any, default=None) -> Any: + return self._entity.get(key, default) + + def __getitem__(self, key: Any) -> Any: + return self._entity[key] + + def walk_ancestors( + self, + continue_test: Callable[[dict], bool] = lambda ent: True + ) -> list[tuple]: + """ + Return a list of tuples of the form: + (hubmap_id entity_dict list-of-ancestors) + where list-of-ancestors is None or a list of tuples of the same form. + + continue_test takes an entity dict as a parameter and returns True if + the descent should continue to the children of that entity, False otherwise. + """ + return walk_ancestors(self._entity, continue_test) + + def list_ancestors(self, + continue_test: Callable[[dict], bool] = lambda ent: True, + omit_test: Callable[[dict], bool] = lambda end: False + ) -> list: + """Returns ancestor information in an expanded, non-recursive list""" + return listify(self.walk_ancestors(continue_test), omit_test) + + @property + def is_processed(self): + """Is this a processed dataset, as opposed to a raw (primary) dataset?""" + return is_processed(self._entity) + + def pipeline_steps(self) -> list[dict]: + return pipeline_steps(self._entity) From 4e0946802ceb316bdd3689323ebfcb946ed50cf8 Mon Sep 17 00:00:00 2001 From: Joel Welling Date: Tue, 1 Sep 2026 14:47:48 -0400 Subject: [PATCH 21/36] use WrappedEntity throughout --- .../build_crate_from_dataset.py | 112 ++++++++++++------ src/crate_builder_script/croissant_wrapper.py | 2 +- src/crate_builder_script/extractors.py | 12 ++ 3 files changed, 87 insertions(+), 39 deletions(-) diff --git a/src/crate_builder_script/build_crate_from_dataset.py b/src/crate_builder_script/build_crate_from_dataset.py index 63ae39d..31e5d6c 100644 --- a/src/crate_builder_script/build_crate_from_dataset.py +++ b/src/crate_builder_script/build_crate_from_dataset.py @@ -23,7 +23,7 @@ asset_url, HUBMAP_ORG_ENTITY ) -from extractors import walk_ancestors, is_processed +from extractors import WrappedEntity from croissant_wrapper import CroissantWrapper logging.basicConfig( @@ -161,21 +161,63 @@ def build_contributors(crate: ROCrate, contributors: List[dict]) -> List[Context return ent_l -def count_versions(ds_info: dict) -> int: - if "previous_revision_uuid" in ds_info: - return ( - count_versions(fetch_entity_info(ds_info["previous_revision_uuid"])) - + 1 - ) - else: - return 1 +def build_hubmap_org_entity(crate: ROCrate) -> ContextEntity: + hubmap_org = crate.add(ContextEntity( + crate, + HUBMAP_ORG_ENTITY, + properties={ + "@type": "Organization", + "name": "HuBMAP Consortium", + "url": HUBMAP_ORG_ENTITY + } + )) + return hubmap_org -def build_derived_prov(ds_info: dict, crate: ROCrate) -> ContextEntity: - prov_chain = walk_ancestors( - ds_info, - lambda d: d["entity_type"] == "Dataset" +def build_primary_prov(ds_entity: WrappedEntity, crate: ROCrate) -> ContextEntity: + """ + This is a dummy routine for now. It needs enough structure to define + crate.mainEntity. + """ + + hubmap_org = build_hubmap_org_entity(crate) + agent = crate.add(SoftwareApplication( + crate, + "HuBMAP Process", + properties={"parentOrganization": {"@id": hubmap_org.id}} + )) + python_lang = crate.add(ComputerLanguage( # TODO this is surely wrong here + crate, + identifier="https://python.org", + properties={"name": "Python", "version": "3.11", "url": "https://python.org"} + )) + workflow_file = crate.add_file( + "https://github.com/hubmapconsortium/data-containers/blob/85441770eafe7da487b35d76112ec099d7b5b8f7/src/crate_builder_script/build_crate_from_dataset.py", + properties={ + "@type": ["File", "SoftwareSourceCode", "ComputationalWorkflow"], + "name":"build_crate_from_dataset.py", + "description": "this should be the dag description. But how to reference CWLs?", + "programmingLanguage": {"@id": python_lang.id} + } ) + crate.mainEntity = workflow_file + props = { + "@id": "#some_workflow", + "@type": "CreateAction", + "name": "the-create-action", + "startTime": datetime.now().isoformat(), + "endTime": datetime.now().isoformat(), + "agent": {"@id": agent.id}, + "instrument": {"@id": workflow_file.id}, + "object": [], + "result": [{"@id": "./"}], # the target dataset + "actionStatus": "CompletedActionStatus" + } + return ContextEntity(crate, identifier=props["@id"], properties=props) + + +def build_derived_prov(ds_entity: WrappedEntity, crate: ROCrate) -> ContextEntity: + prov_chain = ds_entity.walk_ancestors(lambda d: d["entity_type"] == "Dataset") assert len(prov_chain) == 1 assert len(prov_chain[0]) == 3 hubmap_id, parent_chain = prov_chain[0][0], prov_chain[0][2] @@ -191,15 +233,7 @@ def build_derived_prov(ds_info: dict, crate: ROCrate) -> ContextEntity: } )) parent_id_list.append(parent_info["doi_url"]) - hubmap_org = crate.add(ContextEntity( - crate, - HUBMAP_ORG_ENTITY, - properties={ - "@type": "Organization", - "name": "HuBMAP Consortium", - "url": HUBMAP_ORG_ENTITY - } - )) + hubmap_org = build_hubmap_org_entity(crate) agent = crate.add(SoftwareApplication( crate, "HuBMAP Process", @@ -317,7 +351,7 @@ def main() -> None: logging.getLogger("urllib3").setLevel(logging.DEBUG) logging.getLogger("api_calls").setLevel(logging.DEBUG) logging.getLogger("croissant_wrapper").setLevel(logging.DEBUG) - ds_info = fetch_entity_info(target_id) + ds_entity = WrappedEntity(fetch_entity_info(target_id)) uuid_files = fetch_uuid_files_info(target_id) blk_idx = {} @@ -336,19 +370,19 @@ def main() -> None: crate.metadata.extra_contexts.append("https://w3id.org/ro/terms/workflow-run/context") - wrapped_croissant = CroissantWrapper(target_id, ds_info["title"]) + wrapped_croissant = CroissantWrapper(target_id, ds_entity["title"]) crate.root_dataset["name"] = target_id - crate.root_dataset["description"] = ds_info["title"] - if "doi_url" in ds_info: - doi_url = ds_info["doi_url"] + crate.root_dataset["description"] = ds_entity["title"] + if "doi_url" in ds_entity: + doi_url = ds_entity["doi_url"] crate.root_dataset["identifier"] = doi_url crate.root_dataset["sameAs"] = doi_url wrapped_croissant.cite_as = doi_url - if "published_timestamp" in ds_info: + if "published_timestamp" in ds_entity: date_published = str( - datetime.fromtimestamp(ds_info["published_timestamp"] // 1000).astimezone( + datetime.fromtimestamp(ds_entity["published_timestamp"] // 1000).astimezone( timezone.utc ) ) @@ -361,36 +395,38 @@ def main() -> None: crate.root_dataset["funder"] = crate.add(build_funder_entity(crate)) - ds_version = count_versions(ds_info) + ds_version = ds_entity.count_versions() crate.root_dataset["version"] = ds_version wrapped_croissant.version = ds_version - if contributors := ds_info.get("contributors"): + if contributors := ds_entity.get("contributors"): crate.add(build_pi_entity(crate)) ent_l = build_contributors(crate, contributors) [crate.add(ent) for ent in ent_l] crate.root_dataset["contributor"] = ent_l - if is_processed(ds_info): - crate.add(build_derived_prov(ds_info, crate)) + if ds_entity.is_processed: + crate.add(build_derived_prov(ds_entity, crate)) + else: + crate.add(build_primary_prov(ds_entity, crate)) - if "files" in ds_info: + if "files" in ds_entity: # This is a derived dataset- include only data products and qa_qc files - for fl in ds_info["files"]: + for fl in ds_entity["files"]: if fl["is_data_product"] or fl["is_qa_qc"] or include_all_files: LOGGER.debug(f"Adding {fl['rel_path']}") crate.add_file( - asset_url(ds_info["uuid"], fl["rel_path"]), + asset_url(ds_entity["uuid"], fl["rel_path"]), validate_url=True ) - wrapped_croissant.add_file(ds_info["uuid"], + wrapped_croissant.add_file(ds_entity["uuid"], fl, blk_idx.get(fl["rel_path"])) else: LOGGER.debug(f"{fl['rel_path']} is not a data product") else: for fl_blk in blk_idx.values(): crate.add_file( - asset_url(ds_info["uuid"], fl_blk["path"]), + asset_url(ds_entity["uuid"], fl_blk["path"]), validate_url=True ) # We have no descriptive info for these files, so it's hard diff --git a/src/crate_builder_script/croissant_wrapper.py b/src/crate_builder_script/croissant_wrapper.py index f5a2284..9efd375 100644 --- a/src/crate_builder_script/croissant_wrapper.py +++ b/src/crate_builder_script/croissant_wrapper.py @@ -6,7 +6,7 @@ from api_calls import fetch_entity_info, asset_url, HUBMAP -from extractors import WrappedEntity, walk_ancestors, listify, is_processed, pipeline_steps +from extractors import WrappedEntity LOGGER = logging.getLogger(__name__) diff --git a/src/crate_builder_script/extractors.py b/src/crate_builder_script/extractors.py index ab92df0..0ab5ab1 100644 --- a/src/crate_builder_script/extractors.py +++ b/src/crate_builder_script/extractors.py @@ -108,6 +108,9 @@ def get(self, key: Any, default=None) -> Any: def __getitem__(self, key: Any) -> Any: return self._entity[key] + def __contains__(self, key: Any) -> bool: + return key in self._entity + def walk_ancestors( self, continue_test: Callable[[dict], bool] = lambda ent: True @@ -136,3 +139,12 @@ def is_processed(self): def pipeline_steps(self) -> list[dict]: return pipeline_steps(self._entity) + + def count_versions(self) -> int: + if "previous_revision_uuid" in self: + prev_ent = WrappedEntity(fetch_entity_info(self["previous_revision_uuid"])) + return (prev_ent.count_versions() + 1) + else: + return 1 + + From d7388ca2b97b729f414d07a88eb5fd4fa1305b97 Mon Sep 17 00:00:00 2001 From: Joel Welling Date: Tue, 1 Sep 2026 16:10:07 -0400 Subject: [PATCH 22/36] working on workflow steps. WIP. --- .../build_crate_from_dataset.py | 27 ++++++++++++++++--- 1 file changed, 24 insertions(+), 3 deletions(-) diff --git a/src/crate_builder_script/build_crate_from_dataset.py b/src/crate_builder_script/build_crate_from_dataset.py index 31e5d6c..06a3bf4 100644 --- a/src/crate_builder_script/build_crate_from_dataset.py +++ b/src/crate_builder_script/build_crate_from_dataset.py @@ -35,11 +35,15 @@ DEFAULT_OUTPUT_PATH = "/tmp/crate_test" CROISSANT_FILENAME = "croissant.json" +APACHE_ORG_ENTITY = "https://apache.org" + # Externally defined identifiers NIH_URI = "https://ror.org/01cwqze88" ORCID_URI = "https://orcid.org" OBOLIB_URI = "http://purl.obolibrary.org/obo" +AIRFLOW_VERSION = "2.11.0" + ############### # Notes- # - count_versions() is essentially untested, for lack of an example @@ -51,6 +55,9 @@ # - Writing a croissant for a file in an unpublished dataset results in an error at # validation time because the file block information from uuid-api has not yet been # set so the sha256 code is not known. +# - We need the Airflow version to write a valid provenance block, but +# AFAIK it is not maintained in the entity information. Likewise, some +# datasets were produced with python versions before 3.11. # - unpublished examples: # TARGET_ID = "HBM567.VCBK.562" # TARGET_ID = "HBM487.HJZB.546" # primary dataset @@ -220,7 +227,7 @@ def build_derived_prov(ds_entity: WrappedEntity, crate: ROCrate) -> ContextEntit prov_chain = ds_entity.walk_ancestors(lambda d: d["entity_type"] == "Dataset") assert len(prov_chain) == 1 assert len(prov_chain[0]) == 3 - hubmap_id, parent_chain = prov_chain[0][0], prov_chain[0][2] + parent_chain = prov_chain[0][2] parent_id_list = [] for tuple in parent_chain: parent_id, parent_info = tuple[0:2] @@ -234,10 +241,24 @@ def build_derived_prov(ds_entity: WrappedEntity, crate: ROCrate) -> ContextEntit )) parent_id_list.append(parent_info["doi_url"]) hubmap_org = build_hubmap_org_entity(crate) + apache_org = crate.add(ContextEntity( + crate, + APACHE_ORG_ENTITY, + properties={ + "@type": "Organization", + "name": "Apache Software Foundation", + "url": APACHE_ORG_ENTITY + } + )) agent = crate.add(SoftwareApplication( crate, - "HuBMAP Process", - properties={"parentOrganization": {"@id": hubmap_org.id}} + "Apache Airflow", + properties={ + "version": AIRFLOW_VERSION, + "description": ("Apache Airflow - A platform to programmatically author," + " schedule, and monitor workflows"), + "publisher": {"@id": hubmap_org.id} + } )) python_lang = crate.add(ComputerLanguage( # TODO this is surely wrong here crate, From a797e397cf98ff2d3dae54b3e088c96e200e54a3 Mon Sep 17 00:00:00 2001 From: Joel Welling Date: Tue, 1 Sep 2026 17:24:09 -0400 Subject: [PATCH 23/36] checkpoint (WIP) --- .../build_crate_from_dataset.py | 41 ++++++++++++++----- 1 file changed, 31 insertions(+), 10 deletions(-) diff --git a/src/crate_builder_script/build_crate_from_dataset.py b/src/crate_builder_script/build_crate_from_dataset.py index 06a3bf4..afee773 100644 --- a/src/crate_builder_script/build_crate_from_dataset.py +++ b/src/crate_builder_script/build_crate_from_dataset.py @@ -223,6 +223,22 @@ def build_primary_prov(ds_entity: WrappedEntity, crate: ROCrate) -> ContextEntit return ContextEntity(crate, identifier=props["@id"], properties=props) +def build_step_entity(step: dict, idx: int, crate: ROCrate) -> ContextEntity: + full_name = f"{step['name']}/{step['cwl']}" + pos = idx + 1 + id_str = f"#step_{pos}" + print(f"STEP idx={idx} {full_name}:\n {pformat(step)}") + return crate.add(ContextEntity( + crate, + id_str, + properties={ + "position": pos, + "@type": "HowToStep", + "name": full_name + } + )) + + def build_derived_prov(ds_entity: WrappedEntity, crate: ROCrate) -> ContextEntity: prov_chain = ds_entity.walk_ancestors(lambda d: d["entity_type"] == "Dataset") assert len(prov_chain) == 1 @@ -265,16 +281,20 @@ def build_derived_prov(ds_entity: WrappedEntity, crate: ROCrate) -> ContextEntit identifier="https://python.org", properties={"name": "Python", "version": "3.11", "url": "https://python.org"} )) - workflow_file = crate.add_file( - "https://github.com/hubmapconsortium/data-containers/blob/85441770eafe7da487b35d76112ec099d7b5b8f7/src/crate_builder_script/build_crate_from_dataset.py", + step_list = [] + for idx, step in enumerate(ds_entity.pipeline_steps()): + step_list.append(build_step_entity(step, idx, crate)) + workflow = crate.add(ContextEntity( + crate, + "#dag_steps", properties={ - "@type": ["File", "SoftwareSourceCode", "ComputationalWorkflow"], - "name":"build_crate_from_dataset.py", - "description": "this should be the dag description. But how to reference CWLs?", - "programmingLanguage": {"@id": python_lang.id} + "@type": "ComputationalWorkflow", + "name":"dag_steps", + "description": "this should be the dag description", + "steps": step_list } - ) - crate.mainEntity = workflow_file + )) + #crate.mainEntity = workflow props = { "@id": "#some_workflow", "@type": "CreateAction", @@ -282,7 +302,7 @@ def build_derived_prov(ds_entity: WrappedEntity, crate: ROCrate) -> ContextEntit "startTime": datetime.now().isoformat(), "endTime": datetime.now().isoformat(), "agent": {"@id": agent.id}, - "instrument": {"@id": workflow_file.id}, + "instrument": {"@id": workflow.id}, "object": [{"@id": this_id} for this_id in parent_id_list], "result": [{"@id": "./"}], # the target dataset "actionStatus": "CompletedActionStatus" @@ -332,7 +352,8 @@ def build_profiles(crate: ROCrate) -> tuple: "version": "1.0" } )) - return (crate_profile, proc_profile, wf_profile, wfc_profile) + # return (crate_profile, proc_profile, wf_profile, wfc_profile) + return (crate_profile, proc_profile) def main() -> None: From 77766acc54300a18101a711690a11f48c11c1638 Mon Sep 17 00:00:00 2001 From: Joel Welling Date: Wed, 2 Sep 2026 17:08:22 -0400 Subject: [PATCH 24/36] fix cwl info for steps --- .../build_crate_from_dataset.py | 67 ++++++++++++++----- 1 file changed, 50 insertions(+), 17 deletions(-) diff --git a/src/crate_builder_script/build_crate_from_dataset.py b/src/crate_builder_script/build_crate_from_dataset.py index afee773..bf4e642 100644 --- a/src/crate_builder_script/build_crate_from_dataset.py +++ b/src/crate_builder_script/build_crate_from_dataset.py @@ -58,6 +58,7 @@ # - We need the Airflow version to write a valid provenance block, but # AFAIK it is not maintained in the entity information. Likewise, some # datasets were produced with python versions before 3.11. +# - the workflow_instance lacks start and end dates # - unpublished examples: # TARGET_ID = "HBM567.VCBK.562" # TARGET_ID = "HBM487.HJZB.546" # primary dataset @@ -223,18 +224,37 @@ def build_primary_prov(ds_entity: WrappedEntity, crate: ROCrate) -> ContextEntit return ContextEntity(crate, identifier=props["@id"], properties=props) -def build_step_entity(step: dict, idx: int, crate: ROCrate) -> ContextEntity: - full_name = f"{step['name']}/{step['cwl']}" +def build_cwl_entity(crate: ROCrate) -> ContextEntity: + props = { + "@id": "https://w3id.org/workflowhub/workflow-ro-crate#cwl", + "@type": "ComputerLanguage", + "name": "Common Workflow Language", + "alternateName": "CWL", + "identifier": { "@id": "https://w3id.org/cwl/v1.2/" }, + "url": "https://www.commonwl.org/" + } + return crate.add(ContextEntity( + crate, identifier=props["@id"], properties=props + )) + + +def build_step_entity(step: dict, idx: int, cwl_entity: ContextEntity, + crate: ROCrate) -> ContextEntity: pos = idx + 1 id_str = f"#step_{pos}" - print(f"STEP idx={idx} {full_name}:\n {pformat(step)}") + hash = step["commit"] return crate.add(ContextEntity( crate, id_str, properties={ "position": pos, "@type": "HowToStep", - "name": full_name + "name": step["cwl"], + "description": step["name"], + "version": hash, + "url": step["repo"], + "codeRepository": step["repo"], + "programmingLanguage": cwl_entity.id } )) @@ -266,14 +286,14 @@ def build_derived_prov(ds_entity: WrappedEntity, crate: ROCrate) -> ContextEntit "url": APACHE_ORG_ENTITY } )) - agent = crate.add(SoftwareApplication( + airflow = crate.add(SoftwareApplication( crate, "Apache Airflow", properties={ "version": AIRFLOW_VERSION, "description": ("Apache Airflow - A platform to programmatically author," " schedule, and monitor workflows"), - "publisher": {"@id": hubmap_org.id} + "publisher": {"@id": apache_org.id} } )) python_lang = crate.add(ComputerLanguage( # TODO this is surely wrong here @@ -281,27 +301,40 @@ def build_derived_prov(ds_entity: WrappedEntity, crate: ROCrate) -> ContextEntit identifier="https://python.org", properties={"name": "Python", "version": "3.11", "url": "https://python.org"} )) - step_list = [] - for idx, step in enumerate(ds_entity.pipeline_steps()): - step_list.append(build_step_entity(step, idx, crate)) + cwl_entity = build_cwl_entity(crate) + all_steps = ds_entity.pipeline_steps() + assert all_steps[0]["name"] == "ingest-pipeline" and not all_steps[0]["cwl"] + ingest_pipeline_info = all_steps[0] + hash = ingest_pipeline_info["commit"] + cwl_steps = all_steps[1:] + assert all(step["cwl"] for step in cwl_steps), "Found a step which is not CWL?" + step_list = [build_step_entity(step, idx, cwl_entity, crate) + for idx, step in enumerate(cwl_steps)] workflow = crate.add(ContextEntity( crate, - "#dag_steps", + "#workflow", properties={ - "@type": "ComputationalWorkflow", - "name":"dag_steps", - "description": "this should be the dag description", - "steps": step_list + "@type": ["ComputationalWorkflow", "SoftwareApplication"], + "name":"ingest-pipeline dag workflow", + "description": "Processing steps implemeneted by an ingest-pipeline DAG", + "steps": step_list, + "version": hash, + "url": ingest_pipeline_info["repo"], + "codeRepository": ingest_pipeline_info["repo"], + "downloadUrl": f"{ingest_pipeline_info['repo']}/archive/{hash}.zip", + "publisher": {"@id": hubmap_org.id}, + "programmingLanguage": {"@id": python_lang.id}, + "softwareRequirements": {"@id": airflow.id} } )) #crate.mainEntity = workflow props = { - "@id": "#some_workflow", + "@id": "#workflow_instance", "@type": "CreateAction", "name": "the-create-action", - "startTime": datetime.now().isoformat(), + "startTime": datetime.now().isoformat(), # TODO: do I have these values? "endTime": datetime.now().isoformat(), - "agent": {"@id": agent.id}, + "agent": {"@id": workflow.id}, "instrument": {"@id": workflow.id}, "object": [{"@id": this_id} for this_id in parent_id_list], "result": [{"@id": "./"}], # the target dataset From 655794ebceca36cbe90dd17080c380109fb00855 Mon Sep 17 00:00:00 2001 From: Joel Welling Date: Wed, 2 Sep 2026 17:20:08 -0400 Subject: [PATCH 25/36] hardcode CWL version --- src/crate_builder_script/build_crate_from_dataset.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/crate_builder_script/build_crate_from_dataset.py b/src/crate_builder_script/build_crate_from_dataset.py index bf4e642..3af37e2 100644 --- a/src/crate_builder_script/build_crate_from_dataset.py +++ b/src/crate_builder_script/build_crate_from_dataset.py @@ -43,6 +43,7 @@ OBOLIB_URI = "http://purl.obolibrary.org/obo" AIRFLOW_VERSION = "2.11.0" +CWL_VERSION = "v1.1" ############### # Notes- @@ -57,7 +58,9 @@ # set so the sha256 code is not known. # - We need the Airflow version to write a valid provenance block, but # AFAIK it is not maintained in the entity information. Likewise, some -# datasets were produced with python versions before 3.11. +# datasets were produced with python versions before 3.11. Likewise, I've +# hard-coded the CWL version, but ours is actually modified- the CWL language +# definition entity should point at ours rather than at default CWL. # - the workflow_instance lacks start and end dates # - unpublished examples: # TARGET_ID = "HBM567.VCBK.562" @@ -230,7 +233,7 @@ def build_cwl_entity(crate: ROCrate) -> ContextEntity: "@type": "ComputerLanguage", "name": "Common Workflow Language", "alternateName": "CWL", - "identifier": { "@id": "https://w3id.org/cwl/v1.2/" }, + "identifier": { "@id": f"https://w3id.org/cwl/{CWL_VERSION}/" }, "url": "https://www.commonwl.org/" } return crate.add(ContextEntity( From 119a6ec3c2ba9e0a0f96fd83cc026c4c3c6408dc Mon Sep 17 00:00:00 2001 From: Joel Welling Date: Wed, 2 Sep 2026 17:34:45 -0400 Subject: [PATCH 26/36] add full workflow description --- src/crate_builder_script/build_crate_from_dataset.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/crate_builder_script/build_crate_from_dataset.py b/src/crate_builder_script/build_crate_from_dataset.py index 3af37e2..41a884e 100644 --- a/src/crate_builder_script/build_crate_from_dataset.py +++ b/src/crate_builder_script/build_crate_from_dataset.py @@ -309,6 +309,10 @@ def build_derived_prov(ds_entity: WrappedEntity, crate: ROCrate) -> ContextEntit assert all_steps[0]["name"] == "ingest-pipeline" and not all_steps[0]["cwl"] ingest_pipeline_info = all_steps[0] hash = ingest_pipeline_info["commit"] + desc = (ds_entity["ingest_metadata"]["workflow_description"] + if "ingest_metadata" in ds_entity + and "workflow_description" in ds_entity["ingest_metadata"] + else "Processing steps implemeneted by an ingest-pipeline DAG") cwl_steps = all_steps[1:] assert all(step["cwl"] for step in cwl_steps), "Found a step which is not CWL?" step_list = [build_step_entity(step, idx, cwl_entity, crate) @@ -319,7 +323,7 @@ def build_derived_prov(ds_entity: WrappedEntity, crate: ROCrate) -> ContextEntit properties={ "@type": ["ComputationalWorkflow", "SoftwareApplication"], "name":"ingest-pipeline dag workflow", - "description": "Processing steps implemeneted by an ingest-pipeline DAG", + "description": desc, "steps": step_list, "version": hash, "url": ingest_pipeline_info["repo"], From 16f23b5c3cefe61530c45fb4c4dcb51bbe52b6f4 Mon Sep 17 00:00:00 2001 From: Joel Welling Date: Thu, 3 Sep 2026 11:52:29 -0400 Subject: [PATCH 27/36] working but incomplete primary ds provenance --- .../build_crate_from_dataset.py | 71 ++++++++----------- 1 file changed, 30 insertions(+), 41 deletions(-) diff --git a/src/crate_builder_script/build_crate_from_dataset.py b/src/crate_builder_script/build_crate_from_dataset.py index 41a884e..64dfcb0 100644 --- a/src/crate_builder_script/build_crate_from_dataset.py +++ b/src/crate_builder_script/build_crate_from_dataset.py @@ -187,39 +187,29 @@ def build_hubmap_org_entity(crate: ROCrate) -> ContextEntity: def build_primary_prov(ds_entity: WrappedEntity, crate: ROCrate) -> ContextEntity: """ - This is a dummy routine for now. It needs enough structure to define - crate.mainEntity. + This is largely a dummy routine for now. """ - hubmap_org = build_hubmap_org_entity(crate) agent = crate.add(SoftwareApplication( crate, "HuBMAP Process", properties={"parentOrganization": {"@id": hubmap_org.id}} )) - python_lang = crate.add(ComputerLanguage( # TODO this is surely wrong here + instrument_model = ds_entity["metadata"].get("acquisition_instrument_model", "") + instrument_entity = crate.add(ContextEntity( crate, - identifier="https://python.org", - properties={"name": "Python", "version": "3.11", "url": "https://python.org"} - )) - workflow_file = crate.add_file( - "https://github.com/hubmapconsortium/data-containers/blob/85441770eafe7da487b35d76112ec099d7b5b8f7/src/crate_builder_script/build_crate_from_dataset.py", + "#instrument", properties={ - "@type": ["File", "SoftwareSourceCode", "ComputationalWorkflow"], - "name":"build_crate_from_dataset.py", - "description": "this should be the dag description. But how to reference CWLs?", - "programmingLanguage": {"@id": python_lang.id} + "@type": ["IndividualProduct", "ComputationalWorkflow"], + "name": instrument_model, } - ) - crate.mainEntity = workflow_file + )) props = { "@id": "#some_workflow", "@type": "CreateAction", "name": "the-create-action", - "startTime": datetime.now().isoformat(), - "endTime": datetime.now().isoformat(), "agent": {"@id": agent.id}, - "instrument": {"@id": workflow_file.id}, + "instrument": {"@id": instrument_entity.id}, "object": [], "result": [{"@id": "./"}], # the target dataset "actionStatus": "CompletedActionStatus" @@ -334,13 +324,10 @@ def build_derived_prov(ds_entity: WrappedEntity, crate: ROCrate) -> ContextEntit "softwareRequirements": {"@id": airflow.id} } )) - #crate.mainEntity = workflow props = { "@id": "#workflow_instance", "@type": "CreateAction", "name": "the-create-action", - "startTime": datetime.now().isoformat(), # TODO: do I have these values? - "endTime": datetime.now().isoformat(), "agent": {"@id": workflow.id}, "instrument": {"@id": workflow.id}, "object": [{"@id": this_id} for this_id in parent_id_list], @@ -373,25 +360,27 @@ def build_profiles(crate: ROCrate) -> tuple: "version": "0.5" } )) - wf_profile = crate.add(ContextEntity( - crate, - "https://w3id.org/ro/wfrun/workflow/0.5", - properties={ - "@type": ["CreativeWork", "Profile"], - "name": "Workflow Run Crate Profile", - "version": "0.5", - # "isProfileOf": {"@id": proc_profile.id} - } - )) - wfc_profile = crate.add(ContextEntity( - crate, - "https://w3id.org/workflowhub/workflow-ro-crate/1.0", - properties={ - "@type": ["CreativeWork", "Profile"], - "name": "Workflow Run RO-Crate", - "version": "1.0" - } - )) + # # These two profiles are needed for full software workflows + # + # wf_profile = crate.add(ContextEntity( + # crate, + # "https://w3id.org/ro/wfrun/workflow/0.5", + # properties={ + # "@type": ["CreativeWork", "Profile"], + # "name": "Workflow Run Crate Profile", + # "version": "0.5", + # # "isProfileOf": {"@id": proc_profile.id} + # } + # )) + # wfc_profile = crate.add(ContextEntity( + # crate, + # "https://w3id.org/workflowhub/workflow-ro-crate/1.0", + # properties={ + # "@type": ["CreativeWork", "Profile"], + # "name": "Workflow Run RO-Crate", + # "version": "1.0" + # } + # )) # return (crate_profile, proc_profile, wf_profile, wfc_profile) return (crate_profile, proc_profile) @@ -529,7 +518,7 @@ def main() -> None: "url": "https://docs.mlcommons.org/croissant/docs/crossant-spec-1.0.html" } )) - croissant_crate_file = crate.add_file( + crate.add_file( os.path.join(tmpdir.name, CROISSANT_FILENAME), properties={ "name": "Croissant Metadata Descriptor", From efa30a0bedd4e57336d448ab334e9184caa6a1c7 Mon Sep 17 00:00:00 2001 From: Joel Welling Date: Thu, 3 Sep 2026 12:21:06 -0400 Subject: [PATCH 28/36] black/lint --- src/crate_builder_script/api_calls.py | 20 +- .../build_crate_from_dataset.py | 341 ++++++++++-------- src/crate_builder_script/croissant_wrapper.py | 178 +++++---- src/crate_builder_script/extractors.py | 35 +- src/cwl_example/build_rocrate.py | 2 + 5 files changed, 323 insertions(+), 253 deletions(-) diff --git a/src/crate_builder_script/api_calls.py b/src/crate_builder_script/api_calls.py index 5504850..2ebd93f 100644 --- a/src/crate_builder_script/api_calls.py +++ b/src/crate_builder_script/api_calls.py @@ -17,26 +17,34 @@ HUBMAP_ORG_ENTITY = HUBMAP # for lack of a better choice + def fetch_entity_info(target_id: str) -> dict[str, Any]: resp = requests.get( ENTITY_API + f"/entities/{target_id}", - headers={"Authorization": f"Bearer {AUTH_TOK}"} + headers={"Authorization": f"Bearer {AUTH_TOK}"}, ) resp.raise_for_status() ds_info = resp.json() LOGGER.debug("TOP LEVEL for %s:\n%s", target_id, pformat(ds_info, depth=1)) - LOGGER.debug("INGEST METADATA:\n%s", pformat(ds_info.get("ingest_metadata", {}), - depth=2)) + LOGGER.debug( + "INGEST METADATA:\n%s", pformat(ds_info.get("ingest_metadata", {}), depth=2) + ) LOGGER.debug("METADATA:\n%s", pformat(ds_info.get("metadata", {}), depth=2)) - LOGGER.debug("DIRECT ANCESTORS:\n%s", pformat(ds_info.get("direct_ancestors"), depth=2)) - LOGGER.debug("DIRECT ANCESTOR:\n%s", pformat(ds_info.get("direct_ancestor"), depth=2)) + LOGGER.debug( + "DIRECT ANCESTORS:\n%s", pformat(ds_info.get("direct_ancestors"), depth=2) + ) + LOGGER.debug( + "DIRECT ANCESTOR:\n%s", pformat(ds_info.get("direct_ancestor"), depth=2) + ) if "direct_ancestors" in ds_info: first_ancestor = ds_info["direct_ancestors"][0] elif "direct_ancestor" in ds_info: first_ancestor = ds_info["direct_ancestor"] else: first_ancestor = {} - LOGGER.debug("ANCESTOR INGEST MD\n%s", pformat(first_ancestor.get("ingest_metadata", {}))) + LOGGER.debug( + "ANCESTOR INGEST MD\n%s", pformat(first_ancestor.get("ingest_metadata", {})) + ) LOGGER.debug("ANCESTOR MD\n%s", pformat(first_ancestor.get("metadata", {}))) return ds_info diff --git a/src/crate_builder_script/build_crate_from_dataset.py b/src/crate_builder_script/build_crate_from_dataset.py index 64dfcb0..3c449ca 100644 --- a/src/crate_builder_script/build_crate_from_dataset.py +++ b/src/crate_builder_script/build_crate_from_dataset.py @@ -5,26 +5,16 @@ from collections import defaultdict from datetime import datetime, timezone from pprint import pformat, pprint -from typing import List from tempfile import TemporaryDirectory +from typing import List -from rocrate.model import ( - ContextEntity, - Person, - Dataset, - ComputerLanguage, - SoftwareApplication -) -from rocrate.rocrate import ROCrate - -from api_calls import ( - fetch_entity_info, - fetch_uuid_files_info, - asset_url, - HUBMAP_ORG_ENTITY -) -from extractors import WrappedEntity +from api_calls import (HUBMAP_ORG_ENTITY, asset_url, fetch_entity_info, + fetch_uuid_files_info) from croissant_wrapper import CroissantWrapper +from extractors import WrappedEntity +from rocrate.model import (ComputerLanguage, ContextEntity, Dataset, Person, + SoftwareApplication) +from rocrate.rocrate import ROCrate logging.basicConfig( level=logging.INFO, @@ -173,15 +163,17 @@ def build_contributors(crate: ROCrate, contributors: List[dict]) -> List[Context def build_hubmap_org_entity(crate: ROCrate) -> ContextEntity: - hubmap_org = crate.add(ContextEntity( - crate, - HUBMAP_ORG_ENTITY, - properties={ - "@type": "Organization", - "name": "HuBMAP Consortium", - "url": HUBMAP_ORG_ENTITY - } - )) + hubmap_org = crate.add( + ContextEntity( + crate, + HUBMAP_ORG_ENTITY, + properties={ + "@type": "Organization", + "name": "HuBMAP Consortium", + "url": HUBMAP_ORG_ENTITY, + }, + ) + ) return hubmap_org @@ -190,20 +182,24 @@ def build_primary_prov(ds_entity: WrappedEntity, crate: ROCrate) -> ContextEntit This is largely a dummy routine for now. """ hubmap_org = build_hubmap_org_entity(crate) - agent = crate.add(SoftwareApplication( - crate, - "HuBMAP Process", - properties={"parentOrganization": {"@id": hubmap_org.id}} - )) + agent = crate.add( + SoftwareApplication( + crate, + "HuBMAP Process", + properties={"parentOrganization": {"@id": hubmap_org.id}}, + ) + ) instrument_model = ds_entity["metadata"].get("acquisition_instrument_model", "") - instrument_entity = crate.add(ContextEntity( - crate, - "#instrument", - properties={ - "@type": ["IndividualProduct", "ComputationalWorkflow"], - "name": instrument_model, - } - )) + instrument_entity = crate.add( + ContextEntity( + crate, + "#instrument", + properties={ + "@type": ["IndividualProduct", "ComputationalWorkflow"], + "name": instrument_model, + }, + ) + ) props = { "@id": "#some_workflow", "@type": "CreateAction", @@ -212,7 +208,7 @@ def build_primary_prov(ds_entity: WrappedEntity, crate: ROCrate) -> ContextEntit "instrument": {"@id": instrument_entity.id}, "object": [], "result": [{"@id": "./"}], # the target dataset - "actionStatus": "CompletedActionStatus" + "actionStatus": "CompletedActionStatus", } return ContextEntity(crate, identifier=props["@id"], properties=props) @@ -223,107 +219,128 @@ def build_cwl_entity(crate: ROCrate) -> ContextEntity: "@type": "ComputerLanguage", "name": "Common Workflow Language", "alternateName": "CWL", - "identifier": { "@id": f"https://w3id.org/cwl/{CWL_VERSION}/" }, - "url": "https://www.commonwl.org/" + "identifier": {"@id": f"https://w3id.org/cwl/{CWL_VERSION}/"}, + "url": "https://www.commonwl.org/", } - return crate.add(ContextEntity( - crate, identifier=props["@id"], properties=props - )) + return crate.add(ContextEntity(crate, identifier=props["@id"], properties=props)) -def build_step_entity(step: dict, idx: int, cwl_entity: ContextEntity, - crate: ROCrate) -> ContextEntity: +def build_step_entity( + step: dict, idx: int, cwl_entity: ContextEntity, crate: ROCrate +) -> ContextEntity: pos = idx + 1 id_str = f"#step_{pos}" hash = step["commit"] - return crate.add(ContextEntity( - crate, - id_str, - properties={ - "position": pos, - "@type": "HowToStep", - "name": step["cwl"], - "description": step["name"], - "version": hash, - "url": step["repo"], - "codeRepository": step["repo"], - "programmingLanguage": cwl_entity.id - } - )) + return crate.add( + ContextEntity( + crate, + id_str, + properties={ + "position": pos, + "@type": "HowToStep", + "name": step["cwl"], + "description": step["name"], + "version": hash, + "url": step["repo"], + "codeRepository": step["repo"], + "programmingLanguage": cwl_entity.id, + }, + ) + ) def build_derived_prov(ds_entity: WrappedEntity, crate: ROCrate) -> ContextEntity: prov_chain = ds_entity.walk_ancestors(lambda d: d["entity_type"] == "Dataset") assert len(prov_chain) == 1 - assert len(prov_chain[0]) == 3 + assert len(prov_chain[0]) == 3 parent_chain = prov_chain[0][2] parent_id_list = [] for tuple in parent_chain: parent_id, parent_info = tuple[0:2] - crate.add(Dataset( - crate, - parent_info["doi_url"], - properties={ - "name": parent_id, - "description": parent_info["description"] - } - )) + crate.add( + Dataset( + crate, + parent_info["doi_url"], + properties={ + "name": parent_id, + "description": parent_info["description"], + }, + ) + ) parent_id_list.append(parent_info["doi_url"]) hubmap_org = build_hubmap_org_entity(crate) - apache_org = crate.add(ContextEntity( - crate, - APACHE_ORG_ENTITY, - properties={ - "@type": "Organization", - "name": "Apache Software Foundation", - "url": APACHE_ORG_ENTITY - } - )) - airflow = crate.add(SoftwareApplication( - crate, - "Apache Airflow", - properties={ - "version": AIRFLOW_VERSION, - "description": ("Apache Airflow - A platform to programmatically author," - " schedule, and monitor workflows"), - "publisher": {"@id": apache_org.id} - } - )) - python_lang = crate.add(ComputerLanguage( # TODO this is surely wrong here - crate, - identifier="https://python.org", - properties={"name": "Python", "version": "3.11", "url": "https://python.org"} - )) + apache_org = crate.add( + ContextEntity( + crate, + APACHE_ORG_ENTITY, + properties={ + "@type": "Organization", + "name": "Apache Software Foundation", + "url": APACHE_ORG_ENTITY, + }, + ) + ) + airflow = crate.add( + SoftwareApplication( + crate, + "Apache Airflow", + properties={ + "version": AIRFLOW_VERSION, + "description": ( + "Apache Airflow - A platform to programmatically author," + " schedule, and monitor workflows" + ), + "publisher": {"@id": apache_org.id}, + }, + ) + ) + python_lang = crate.add( + ComputerLanguage( # TODO this is surely wrong here + crate, + identifier="https://python.org", + properties={ + "name": "Python", + "version": "3.11", + "url": "https://python.org", + }, + ) + ) cwl_entity = build_cwl_entity(crate) all_steps = ds_entity.pipeline_steps() assert all_steps[0]["name"] == "ingest-pipeline" and not all_steps[0]["cwl"] ingest_pipeline_info = all_steps[0] hash = ingest_pipeline_info["commit"] - desc = (ds_entity["ingest_metadata"]["workflow_description"] - if "ingest_metadata" in ds_entity - and "workflow_description" in ds_entity["ingest_metadata"] - else "Processing steps implemeneted by an ingest-pipeline DAG") + desc = ( + ds_entity["ingest_metadata"]["workflow_description"] + if "ingest_metadata" in ds_entity + and "workflow_description" in ds_entity["ingest_metadata"] + else "Processing steps implemeneted by an ingest-pipeline DAG" + ) cwl_steps = all_steps[1:] assert all(step["cwl"] for step in cwl_steps), "Found a step which is not CWL?" - step_list = [build_step_entity(step, idx, cwl_entity, crate) - for idx, step in enumerate(cwl_steps)] - workflow = crate.add(ContextEntity( - crate, - "#workflow", - properties={ - "@type": ["ComputationalWorkflow", "SoftwareApplication"], - "name":"ingest-pipeline dag workflow", - "description": desc, - "steps": step_list, - "version": hash, - "url": ingest_pipeline_info["repo"], - "codeRepository": ingest_pipeline_info["repo"], - "downloadUrl": f"{ingest_pipeline_info['repo']}/archive/{hash}.zip", - "publisher": {"@id": hubmap_org.id}, - "programmingLanguage": {"@id": python_lang.id}, - "softwareRequirements": {"@id": airflow.id} - } - )) + step_list = [ + build_step_entity(step, idx, cwl_entity, crate) + for idx, step in enumerate(cwl_steps) + ] + workflow = crate.add( + ContextEntity( + crate, + "#workflow", + properties={ + "@type": ["ComputationalWorkflow", "SoftwareApplication"], + "name": "ingest-pipeline dag workflow", + "description": desc, + "steps": step_list, + "version": hash, + "url": ingest_pipeline_info["repo"], + "codeRepository": ingest_pipeline_info["repo"], + "downloadUrl": f"{ingest_pipeline_info['repo']}/archive/{hash}.zip", + "publisher": {"@id": hubmap_org.id}, + "programmingLanguage": {"@id": python_lang.id}, + "softwareRequirements": {"@id": airflow.id}, + }, + ) + ) props = { "@id": "#workflow_instance", "@type": "CreateAction", @@ -332,7 +349,7 @@ def build_derived_prov(ds_entity: WrappedEntity, crate: ROCrate) -> ContextEntit "instrument": {"@id": workflow.id}, "object": [{"@id": this_id} for this_id in parent_id_list], "result": [{"@id": "./"}], # the target dataset - "actionStatus": "CompletedActionStatus" + "actionStatus": "CompletedActionStatus", } return ContextEntity(crate, identifier=props["@id"], properties=props) @@ -342,24 +359,28 @@ def build_profiles(crate: ROCrate) -> tuple: Build ContextElements for several profiles needed to describe a workflow. """ base_crate_ctx_id = f"https://w3id.org/ro/crate/{crate.version}" - crate_profile = crate.add(ContextEntity( - crate, - base_crate_ctx_id, - properties={ - "@type": ["CreativeWork", "Profile"], - "name": "RO-Crate Profile", - "version": crate.version - } - )) - proc_profile = crate.add(ContextEntity( - crate, - "https://w3id.org/ro/wfrun/process/0.5", - properties={ - "@type": ["CreativeWork", "Profile"], - "name": "Process Run Crate Profile", - "version": "0.5" - } - )) + crate_profile = crate.add( + ContextEntity( + crate, + base_crate_ctx_id, + properties={ + "@type": ["CreativeWork", "Profile"], + "name": "RO-Crate Profile", + "version": crate.version, + }, + ) + ) + proc_profile = crate.add( + ContextEntity( + crate, + "https://w3id.org/ro/wfrun/process/0.5", + properties={ + "@type": ["CreativeWork", "Profile"], + "name": "Process Run Crate Profile", + "version": "0.5", + }, + ) + ) # # These two profiles are needed for full software workflows # # wf_profile = crate.add(ContextEntity( @@ -387,9 +408,7 @@ def build_profiles(crate: ROCrate) -> tuple: def main() -> None: parser = argparse.ArgumentParser() - parser.add_argument( - "target_id", help="Build an RO-Crate for this dataset" - ) + parser.add_argument("target_id", help="Build an RO-Crate for this dataset") parser.add_argument( "--outdir", "-o", @@ -408,7 +427,7 @@ def main() -> None: parser.add_argument( "--include_all_files", action="store_true", - help="Include all files, even if some are not data product or qa resources" + help="Include all files, even if some are not data product or qa resources", ) args = parser.parse_args() target_id = args.target_id @@ -439,7 +458,9 @@ def main() -> None: for profile in build_profiles(crate): crate.root_dataset.append_to("conformsTo", {"@id": profile.id}) - crate.metadata.extra_contexts.append("https://w3id.org/ro/terms/workflow-run/context") + crate.metadata.extra_contexts.append( + "https://w3id.org/ro/terms/workflow-run/context" + ) wrapped_croissant = CroissantWrapper(target_id, ds_entity["title"]) @@ -487,18 +508,17 @@ def main() -> None: if fl["is_data_product"] or fl["is_qa_qc"] or include_all_files: LOGGER.debug(f"Adding {fl['rel_path']}") crate.add_file( - asset_url(ds_entity["uuid"], fl["rel_path"]), - validate_url=True + asset_url(ds_entity["uuid"], fl["rel_path"]), validate_url=True + ) + wrapped_croissant.add_file( + ds_entity["uuid"], fl, blk_idx.get(fl["rel_path"]) ) - wrapped_croissant.add_file(ds_entity["uuid"], - fl, blk_idx.get(fl["rel_path"])) else: LOGGER.debug(f"{fl['rel_path']} is not a data product") else: for fl_blk in blk_idx.values(): crate.add_file( - asset_url(ds_entity["uuid"], fl_blk["path"]), - validate_url=True + asset_url(ds_entity["uuid"], fl_blk["path"]), validate_url=True ) # We have no descriptive info for these files, so it's hard # to see how we could add them to the Croissant object @@ -508,28 +528,31 @@ def main() -> None: if not os.path.isdir(outdir): os.makedirs(outdir, exist_ok=True) wrapped_croissant.write(os.path.join(tmpdir.name, CROISSANT_FILENAME)) - crate.add(ContextEntity( - crate, - "http://mlcommons.org/croissant/1.0", - properties={ - "@type": ["CreativeWork", "Profile"], - "name": "MLCommons Croissant Format Specification", - "version": "1.0", - "url": "https://docs.mlcommons.org/croissant/docs/crossant-spec-1.0.html" - } - )) + crate.add( + ContextEntity( + crate, + "http://mlcommons.org/croissant/1.0", + properties={ + "@type": ["CreativeWork", "Profile"], + "name": "MLCommons Croissant Format Specification", + "version": "1.0", + "url": "https://docs.mlcommons.org/croissant/docs/crossant-spec-1.0.html", + }, + ) + ) crate.add_file( os.path.join(tmpdir.name, CROISSANT_FILENAME), properties={ "name": "Croissant Metadata Descriptor", "description": "Machine learning data-loading configurations for this dataset.", "encodingFormat": "application/ld+json", - "conformsTo": {"@id": "http://mlcommons.org/croissant/1.0"} - } + "conformsTo": {"@id": "http://mlcommons.org/croissant/1.0"}, + }, ) crate.write_zip(os.path.join(outdir, f"{target_id}_crate.zip")) tmpdir.cleanup() + if __name__ == "__main__": main() diff --git a/src/crate_builder_script/croissant_wrapper.py b/src/crate_builder_script/croissant_wrapper.py index 9efd375..e972460 100644 --- a/src/crate_builder_script/croissant_wrapper.py +++ b/src/crate_builder_script/croissant_wrapper.py @@ -3,33 +3,37 @@ from pprint import pformat import mlcroissant as mlc - -from api_calls import fetch_entity_info, asset_url, HUBMAP - +from api_calls import HUBMAP, asset_url, fetch_entity_info from extractors import WrappedEntity LOGGER = logging.getLogger(__name__) EDAM_INFO = { - "EDAM_1.24.format_3727" : {"desc":"tiff", "mime":"image/tiff"}, - "EDAM_1.24.format_3464" : {"desc":"json", "mime":"application/json"}, - "EDAM_1.24.format_3508" : {"desc":"pdf", "mime":"application/pdf"}, - "EDAM_1.24.format_3590" : {"desc":"hdf5", "mime":"application/x-hdf5"}, - "EDAM_1.24.format_3987" : {"desc":"zip", "mime":"application/zip"}, - "EDAM_1.24.format_3752" : {"desc":"csv", "mime":"text/csv"}, - "EDAM_1.24.format_3755" : {"desc":"tsv", "mime":"text/tab-separated-values"}, - "EDAM_1.24.format_3790" : {"desc":"h5ad (anndata)", "mime":"application/x-hdf5"}, - "EDAM_1.24.format_3915" : {"desc":"zarr", "mime":"application/vnd.zarr"}, - "EDAM_1.24.format_4006" : {"desc":"zarr (spatialdata)", "mime":"application/vnd.zarr"}, - "EDAM_1.24.data_3671" : {"desc":"plain text", "mime":"text/plain"}, - "EDAM_1.24.format_3916" : {"desc":"adjacency matrix", "mime":"text/plain"} + "EDAM_1.24.format_3727": {"desc": "tiff", "mime": "image/tiff"}, + "EDAM_1.24.format_3464": {"desc": "json", "mime": "application/json"}, + "EDAM_1.24.format_3508": {"desc": "pdf", "mime": "application/pdf"}, + "EDAM_1.24.format_3590": {"desc": "hdf5", "mime": "application/x-hdf5"}, + "EDAM_1.24.format_3987": {"desc": "zip", "mime": "application/zip"}, + "EDAM_1.24.format_3752": {"desc": "csv", "mime": "text/csv"}, + "EDAM_1.24.format_3755": {"desc": "tsv", "mime": "text/tab-separated-values"}, + "EDAM_1.24.format_3790": {"desc": "h5ad (anndata)", "mime": "application/x-hdf5"}, + "EDAM_1.24.format_3915": {"desc": "zarr", "mime": "application/vnd.zarr"}, + "EDAM_1.24.format_4006": { + "desc": "zarr (spatialdata)", + "mime": "application/vnd.zarr", + }, + "EDAM_1.24.data_3671": {"desc": "plain text", "mime": "text/plain"}, + "EDAM_1.24.format_3916": {"desc": "adjacency matrix", "mime": "text/plain"}, } def _protocol_dois(md: dict) -> list[str]: dois = [] - for k in ("preparation_protocol_doi", "reagent_prep_protocols_io_doi", - "section_prep_protocols_io_doi"): + for k in ( + "preparation_protocol_doi", + "reagent_prep_protocols_io_doi", + "section_prep_protocols_io_doi", + ): v = (md.get(k) or "").strip() if not v: continue @@ -39,10 +43,14 @@ def _protocol_dois(md: dict) -> list[str]: def _acquisition_activity(entity: WrappedEntity, md: dict) -> dict: def agent(name, role=None, org=False): - n = {"@type": "prov:Organization" if org else "prov:Person", "schema:name": name} + n = { + "@type": "prov:Organization" if org else "prov:Person", + "schema:name": name, + } if role: n["prov:role"] = role return n + assoc = [] if md.get("pi"): assoc.append(agent(md["pi"], role="principal investigator")) @@ -53,8 +61,15 @@ def agent(name, role=None, org=False): act = { "@type": "prov:Activity", "schema:name": f"{entity.get('dataset_type', 'assay')} acquisition", - "hubmap:instrument": " ".join(filter(None, [md.get("acquisition_instrument_vendor"), - md.get("acquisition_instrument_model")])), + "hubmap:instrument": " ".join( + filter( + None, + [ + md.get("acquisition_instrument_vendor"), + md.get("acquisition_instrument_model"), + ], + ) + ), "hubmap:numberOfAntibodies": md.get("number_of_antibodies"), "hubmap:numberOfImagingRounds": md.get("number_of_biomarker_imaging_rounds"), "hubmap:numberOfChannels": md.get("number_of_channels"), @@ -62,8 +77,14 @@ def agent(name, role=None, org=False): } if md.get("execution_datetime"): act["prov:startedAtTime"] = md["execution_datetime"] - protocols = [{"@type": ["prov:Entity", "schema:CreativeWork"], "@id": d, "prov:role": "protocol"} - for d in _protocol_dois(md)] + protocols = [ + { + "@type": ["prov:Entity", "schema:CreativeWork"], + "@id": d, + "prov:role": "protocol", + } + for d in _protocol_dois(md) + ] if protocols: act["prov:used"] = protocols return {k: v for k, v in act.items() if v not in (None, "", [])} @@ -71,19 +92,26 @@ def agent(name, role=None, org=False): def _specimen_chain(entity: WrappedEntity) -> dict: def node(anc): - n = {"@type": "prov:Entity", "@id": HUBMAP + (anc.get("hubmap_id") or ""), - "schema:name": anc.get("hubmap_id"), "hubmap:entityType": anc.get("entity_type"), - "hubmap:sampleCategory": anc.get("sample_category")} + n = { + "@type": "prov:Entity", + "@id": HUBMAP + (anc.get("hubmap_id") or ""), + "schema:name": anc.get("hubmap_id"), + "hubmap:entityType": anc.get("entity_type"), + "hubmap:sampleCategory": anc.get("sample_category"), + } rui = anc.get("rui_location") if rui: r = json.loads(rui) if isinstance(rui, str) else rui n["hubmap:ccfAnnotations"] = r.get("ccf_annotations") - n["hubmap:dimensions"] = {"x": r.get("x_dimension"), "y": r.get("y_dimension"), - "z": r.get("z_dimension"), "unit": r.get("dimension_units")} + n["hubmap:dimensions"] = { + "x": r.get("x_dimension"), + "y": r.get("y_dimension"), + "z": r.get("z_dimension"), + "unit": r.get("dimension_units"), + } return {k: v for k, v in n.items() if v not in (None, "", [])} - ancs = entity.list_ancestors( - omit_test=lambda dct: dct["entity_type"]=="Dataset" - ) + + ancs = entity.list_ancestors(omit_test=lambda dct: dct["entity_type"] == "Dataset") order = {"section": 0, "block": 1, "organ": 2} ancs.sort(key=lambda a: order.get(a.get("sample_category"), 4)) derived = None @@ -98,20 +126,25 @@ def node(anc): def _pipeline_activity(entity: WrappedEntity) -> dict: agents = [] for st in entity.pipeline_steps(): - a = {"@type": ["prov:SoftwareAgent", "schema:SoftwareApplication"], - "schema:name": st["name"] + (f" [{st['cwl']}]" if st["cwl"] else ""), - "schema:codeRepository": st["repo"], "hubmap:commit": st["commit"]} + a = { + "@type": ["prov:SoftwareAgent", "schema:SoftwareApplication"], + "schema:name": st["name"] + (f" [{st['cwl']}]" if st["cwl"] else ""), + "schema:codeRepository": st["repo"], + "hubmap:commit": st["commit"], + } agents.append({k: v for k, v in a.items() if v}) - act = {"@type": "prov:Activity", "schema:name": "HuBMAP uniform processing pipeline"} + act = { + "@type": "prov:Activity", + "schema:name": "HuBMAP uniform processing pipeline", + } if agents: act["prov:wasAssociatedWith"] = agents return act def build_embedded_provenance( - entity: WrappedEntity, - md: dict | None, - descendants: list | None =None) -> dict: + entity: WrappedEntity, md: dict | None, descendants: list | None = None +) -> dict: """ PROCESSED subject: wasGeneratedBy its own pipeline; wasDerivedFrom the raw parent (which carries the acquisition activity + specimen chain). RAW subject: wasGeneratedBy @@ -119,9 +152,7 @@ def build_embedded_provenance( """ if entity.is_processed: hubmap_id = entity["hubmap_id"] - ancestor_chain = entity.walk_ancestors( - lambda d: d["entity_type"] == "Dataset" - ) + ancestor_chain = entity.walk_ancestors(lambda d: d["entity_type"] == "Dataset") assert len(ancestor_chain) == 1, "internal error walking ancestors" check_id, ignored_entity, ancestors = ancestor_chain[0] assert check_id == hubmap_id @@ -129,39 +160,47 @@ def build_embedded_provenance( raw_entity = WrappedEntity(ancestors[0][1]) raw_md = raw_entity.get("metadata") raw_node = { - "@type": "prov:Entity", "@id": HUBMAP + (raw_entity.get("hubmap_id") or ""), + "@type": "prov:Entity", + "@id": HUBMAP + (raw_entity.get("hubmap_id") or ""), "schema:name": raw_entity.get("hubmap_id"), "hubmap:datasetType": raw_entity.get("dataset_type"), "prov:wasGeneratedBy": _acquisition_activity(raw_entity, raw_md or {}), - "prov:wasDerivedFrom": _specimen_chain(raw_entity) + "prov:wasDerivedFrom": _specimen_chain(raw_entity), + } + return { + "prov:wasGeneratedBy": _pipeline_activity(entity), + "prov:wasDerivedFrom": raw_node, } - return {"prov:wasGeneratedBy": _pipeline_activity(entity), - "prov:wasDerivedFrom": raw_node} provo = {"prov:wasGeneratedBy": _acquisition_activity(entity, md)} if chain := _specimen_chain(entity): provo["prov:wasDerivedFrom"] = chain if descendants: provo["hubmap:hasProcessedDataset"] = [ - {"@type": "prov:Entity", "@id": HUBMAP + (d.get("hubmap_id") or ""), - "schema:name": d.get("hubmap_id"), "hubmap:datasetType": d.get("dataset_type")} - for d in descendants] + { + "@type": "prov:Entity", + "@id": HUBMAP + (d.get("hubmap_id") or ""), + "schema:name": d.get("hubmap_id"), + "hubmap:datasetType": d.get("dataset_type"), + } + for d in descendants + ] return provo -class CroissantWrapper(): +class CroissantWrapper: @classmethod def test(cls, entity_dict: dict) -> None: LOGGER.info( - "Testing CroissantWrapper:\n%s", + "Testing CroissantWrapper:\n%s", pformat( build_embedded_provenance( entity_dict, md=entity_dict.get("metadata"), - descendants=entity_dict.get("direct_descendants", []) + descendants=entity_dict.get("direct_descendants", []), ) - ) + ), ) - + def __init__(self, name: str, description: str): self.name = name self.description = description @@ -174,10 +213,10 @@ def __init__(self, name: str, description: str): def add_file(self, ds_uuid: str, file_info: dict, file_blk: dict | None) -> None: args = { - "id" : file_info["rel_path"], - "name" : file_info["rel_path"], - "description" : file_info["description"], - "content_url" : asset_url(ds_uuid, file_info["rel_path"]) + "id": file_info["rel_path"], + "name": file_info["rel_path"], + "description": file_info["description"], + "content_url": asset_url(ds_uuid, file_info["rel_path"]), } if edam := file_info.get("edam_term"): if edam in EDAM_INFO: @@ -191,19 +230,17 @@ def add_file(self, ds_uuid: str, file_info: dict, file_blk: dict | None) -> None args["sha256"] = file_blk["sha256_checksum"] self.file_objects.append(mlc.FileObject(**args)) - # def add_record_set(self, record_set: mlc.RecordSet): # self.record_sets.append(record_set) - def write(self, croissant_filename: str): args = { - "id" : "croissant-spec", - "name" : self.name, - "description" : self.description, - "distribution" : self.file_objects, - "record_sets" : self.record_sets, - "ctx" : mlc.Context(is_live_dataset=False) + "id": "croissant-spec", + "name": self.name, + "description": self.description, + "distribution": self.file_objects, + "record_sets": self.record_sets, + "ctx": mlc.Context(is_live_dataset=False), } if self.date_published: args["date_published"] = self.date_published @@ -214,20 +251,21 @@ def write(self, croissant_filename: str): if self.cite_as: args["cite_as"] = self.cite_as croissant_meta = mlc.Metadata(**args).to_json() - croissant_meta["@context"].update({ + croissant_meta["@context"].update( + { "hubmap": "https://hubmapconsortium.org/", "mlc": "https://mlcommons.org/", "prov": "http://www.w3.org/ns/prov#", - "schema": "http://schema.org/" - }) + "schema": "http://schema.org/", + } + ) entity = WrappedEntity(fetch_entity_info(self.name)) croissant_meta.update( build_embedded_provenance( entity, - md=entity.get("metadata"), # md - descendants=entity.get("direct_descendants", []) + md=entity.get("metadata"), # md + descendants=entity.get("direct_descendants", []), ) ) with open(croissant_filename, "w", encoding="utf-8") as f: json.dump(croissant_meta, f, indent=2) - diff --git a/src/crate_builder_script/extractors.py b/src/crate_builder_script/extractors.py index 0ab5ab1..e2393a3 100644 --- a/src/crate_builder_script/extractors.py +++ b/src/crate_builder_script/extractors.py @@ -1,16 +1,16 @@ import logging -from pprint import pformat from collections.abc import Callable +from pprint import pformat from typing import Any from api_calls import fetch_entity_info LOGGER = logging.getLogger(__name__) + def walk_ancestors( - entity: dict, - continue_test: Callable[[dict], bool] = lambda ent: True - ) -> list[tuple]: + entity: dict, continue_test: Callable[[dict], bool] = lambda ent: True +) -> list[tuple]: """ Given an entity dictionary, return a list of tuples. Each tuple has the form: @@ -27,8 +27,10 @@ def walk_ancestors( e_id = entity.get("hubmap_id") LOGGER.debug(f"walk_ancestors {e_id} {e_type}") if e_type == "Dataset": - ancs = [walk_ancestors(anc, continue_test) - for anc in entity.get("direct_ancestors", [])] + ancs = [ + walk_ancestors(anc, continue_test) + for anc in entity.get("direct_ancestors", []) + ] if ancs: all_tuples = [] for sub_list in ancs: @@ -48,8 +50,7 @@ def walk_ancestors( if "direct_ancestor" not in entity: LOGGER.debug(f"walk_ancestors fetching dead-end sample {e_id}") entity = fetch_entity_info(e_id) - LOGGER.debug("walk_ancestors fetch yielded:\n%s", - pformat(entity, depth=2)) + LOGGER.debug("walk_ancestors fetch yielded:\n%s", pformat(entity, depth=2)) LOGGER.debug("walk_ancestors end of walk jump result") new_entity = entity.get("direct_ancestor", {}) if e_type == "Donor": @@ -112,9 +113,8 @@ def __contains__(self, key: Any) -> bool: return key in self._entity def walk_ancestors( - self, - continue_test: Callable[[dict], bool] = lambda ent: True - ) -> list[tuple]: + self, continue_test: Callable[[dict], bool] = lambda ent: True + ) -> list[tuple]: """ Return a list of tuples of the form: (hubmap_id entity_dict list-of-ancestors) @@ -125,10 +125,11 @@ def walk_ancestors( """ return walk_ancestors(self._entity, continue_test) - def list_ancestors(self, - continue_test: Callable[[dict], bool] = lambda ent: True, - omit_test: Callable[[dict], bool] = lambda end: False - ) -> list: + def list_ancestors( + self, + continue_test: Callable[[dict], bool] = lambda ent: True, + omit_test: Callable[[dict], bool] = lambda end: False, + ) -> list: """Returns ancestor information in an expanded, non-recursive list""" return listify(self.walk_ancestors(continue_test), omit_test) @@ -143,8 +144,6 @@ def pipeline_steps(self) -> list[dict]: def count_versions(self) -> int: if "previous_revision_uuid" in self: prev_ent = WrappedEntity(fetch_entity_info(self["previous_revision_uuid"])) - return (prev_ent.count_versions() + 1) + return prev_ent.count_versions() + 1 else: return 1 - - diff --git a/src/cwl_example/build_rocrate.py b/src/cwl_example/build_rocrate.py index c0b90d3..108ac1a 100755 --- a/src/cwl_example/build_rocrate.py +++ b/src/cwl_example/build_rocrate.py @@ -1,4 +1,5 @@ #!/bin/env python +"""Demonstrate the use of the --provenance option to cwltool.""" from pathlib import Path from subprocess import run @@ -8,6 +9,7 @@ def main(): + """Demonstrate the use of the --provenance option to cwltool.""" run("cwltool --provenance prov1 hello_world.cwl", shell=True, check=True) run( "cwltool --provenance prov2 hello_world.cwl --message 'hola'", From 0b3e27b5b3bba3b811ea72f39d68f7701626ca7b Mon Sep 17 00:00:00 2001 From: Joel Welling Date: Thu, 3 Sep 2026 14:18:39 -0400 Subject: [PATCH 29/36] spelling error --- src/crate_builder_script/build_crate_from_dataset.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/crate_builder_script/build_crate_from_dataset.py b/src/crate_builder_script/build_crate_from_dataset.py index 3c449ca..487125f 100644 --- a/src/crate_builder_script/build_crate_from_dataset.py +++ b/src/crate_builder_script/build_crate_from_dataset.py @@ -58,6 +58,7 @@ # - published examples: # TARGET_ID = "HBM866.VMBK.952" # TARGET_ID = "HBM473.QLDT.264" # primary dataset +# TARGET_ID = "HBM748.CHWC.963" # derived from snare-seq ############### @@ -314,7 +315,7 @@ def build_derived_prov(ds_entity: WrappedEntity, crate: ROCrate) -> ContextEntit ds_entity["ingest_metadata"]["workflow_description"] if "ingest_metadata" in ds_entity and "workflow_description" in ds_entity["ingest_metadata"] - else "Processing steps implemeneted by an ingest-pipeline DAG" + else "Processing steps implemented by an ingest-pipeline DAG" ) cwl_steps = all_steps[1:] assert all(step["cwl"] for step in cwl_steps), "Found a step which is not CWL?" From dd3f5f9a8bd2fc63684e62965d43d1d05e7ebe50 Mon Sep 17 00:00:00 2001 From: Joel Welling Date: Thu, 3 Sep 2026 14:57:08 -0400 Subject: [PATCH 30/36] black --- .../build_crate_from_dataset.py | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/src/crate_builder_script/build_crate_from_dataset.py b/src/crate_builder_script/build_crate_from_dataset.py index 487125f..f5b5836 100644 --- a/src/crate_builder_script/build_crate_from_dataset.py +++ b/src/crate_builder_script/build_crate_from_dataset.py @@ -8,12 +8,21 @@ from tempfile import TemporaryDirectory from typing import List -from api_calls import (HUBMAP_ORG_ENTITY, asset_url, fetch_entity_info, - fetch_uuid_files_info) +from api_calls import ( + HUBMAP_ORG_ENTITY, + asset_url, + fetch_entity_info, + fetch_uuid_files_info, +) from croissant_wrapper import CroissantWrapper from extractors import WrappedEntity -from rocrate.model import (ComputerLanguage, ContextEntity, Dataset, Person, - SoftwareApplication) +from rocrate.model import ( + ComputerLanguage, + ContextEntity, + Dataset, + Person, + SoftwareApplication, +) from rocrate.rocrate import ROCrate logging.basicConfig( From e30381b91b0c7d20c4868b826fe111365bb96be4 Mon Sep 17 00:00:00 2001 From: Joel Welling Date: Thu, 3 Sep 2026 15:03:07 -0400 Subject: [PATCH 31/36] croissant[dev] requires black 23.11.0 --- src/requirements-dev.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/requirements-dev.txt b/src/requirements-dev.txt index 24b8558..c1b70dc 100644 --- a/src/requirements-dev.txt +++ b/src/requirements-dev.txt @@ -1,4 +1,4 @@ -black==25.1.0 +black==23.11.0 flake8==7.1.0 isort==7.0.0 semantic-version==2.10 From fd9c3e1cbe744dffc96898ea359f132aa110023b Mon Sep 17 00:00:00 2001 From: Joel Welling Date: Thu, 3 Sep 2026 15:52:36 -0400 Subject: [PATCH 32/36] lint --- src/crate_builder_script/api_calls.py | 4 ++ .../build_crate_from_dataset.py | 40 +++++++++---------- src/crate_builder_script/croissant_wrapper.py | 11 ++++- src/crate_builder_script/extractors.py | 26 ++++++++++-- 4 files changed, 56 insertions(+), 25 deletions(-) diff --git a/src/crate_builder_script/api_calls.py b/src/crate_builder_script/api_calls.py index 2ebd93f..e67ab9e 100644 --- a/src/crate_builder_script/api_calls.py +++ b/src/crate_builder_script/api_calls.py @@ -1,3 +1,4 @@ +"""Handle requests to HuBMAP APIs.""" import logging import os from pprint import pformat @@ -19,6 +20,7 @@ def fetch_entity_info(target_id: str) -> dict[str, Any]: + """Fetch a dataset's entity information.""" resp = requests.get( ENTITY_API + f"/entities/{target_id}", headers={"Authorization": f"Bearer {AUTH_TOK}"}, @@ -50,6 +52,7 @@ def fetch_entity_info(target_id: str) -> dict[str, Any]: def fetch_uuid_files_info(target_id: str) -> dict[str, Any]: + """Fetch files informaion from the UUID API.""" resp = requests.get( UUID_API + f"/{target_id}/files", headers={"Authorization": f"Bearer {AUTH_TOK}"}, @@ -60,4 +63,5 @@ def fetch_uuid_files_info(target_id: str) -> dict[str, Any]: def asset_url(uuid: str, rel_path: str) -> str: + """Build the URL by which a file should be available from the ASSETS API.""" return f"{ASSETS_API}/{uuid}/{rel_path}" diff --git a/src/crate_builder_script/build_crate_from_dataset.py b/src/crate_builder_script/build_crate_from_dataset.py index f5b5836..1d00bd3 100644 --- a/src/crate_builder_script/build_crate_from_dataset.py +++ b/src/crate_builder_script/build_crate_from_dataset.py @@ -1,28 +1,19 @@ +"""Build an RO-Crate and a Croissant for a dataset.""" import argparse -import json import logging import os from collections import defaultdict from datetime import datetime, timezone -from pprint import pformat, pprint +# from pprint import pformat, pprint from tempfile import TemporaryDirectory from typing import List -from api_calls import ( - HUBMAP_ORG_ENTITY, - asset_url, - fetch_entity_info, - fetch_uuid_files_info, -) +from api_calls import (HUBMAP_ORG_ENTITY, asset_url, fetch_entity_info, + fetch_uuid_files_info) from croissant_wrapper import CroissantWrapper from extractors import WrappedEntity -from rocrate.model import ( - ComputerLanguage, - ContextEntity, - Dataset, - Person, - SoftwareApplication, -) +from rocrate.model import (ComputerLanguage, ContextEntity, Dataset, Person, + SoftwareApplication) from rocrate.rocrate import ROCrate logging.basicConfig( @@ -72,6 +63,7 @@ def build_funder_entity(crate: ROCrate) -> ContextEntity: + """Build an entity representing the NIH.""" funder_props = { "@id": NIH_URI, "@type": "Organization", @@ -82,6 +74,7 @@ def build_funder_entity(crate: ROCrate) -> ContextEntity: def build_license_entity(crate: ROCrate) -> ContextEntity: + """Build an entity representing the Creative Commons license.""" license_props = { "@type": "CreativeWork", "name": "Creative Commons Atribution 4.0 International", @@ -99,6 +92,7 @@ def build_license_entity(crate: ROCrate) -> ContextEntity: def build_pi_entity(crate: ROCrate) -> ContextEntity: + """Build an entity representing the Principal Investigator role.""" props = { "@id": "#role-principal-investigator", "@type": "Role", @@ -110,6 +104,7 @@ def build_pi_entity(crate: ROCrate) -> ContextEntity: def build_contact_entity(crate: ROCrate) -> ContextEntity: + """Build an entity representing the contact person for the dataset.""" props = { "@id": "#role-contact", "@type": "Role", @@ -126,6 +121,7 @@ def build_contact_entity(crate: ROCrate) -> ContextEntity: def build_ia_entity(crate: ROCrate) -> ContextEntity: + """Build an entity representing the operator for a dataset.""" props = { "@id": "#role-investigative-agent", "@type": "Role", @@ -143,6 +139,7 @@ def build_ia_entity(crate: ROCrate) -> ContextEntity: def build_contributors(crate: ROCrate, contributors: List[dict]) -> List[ContextEntity]: + """Construct a representation of contributors information.""" ent_l = [] role_d = {} role_list_d = defaultdict(list) @@ -173,6 +170,7 @@ def build_contributors(crate: ROCrate, contributors: List[dict]) -> List[Context def build_hubmap_org_entity(crate: ROCrate) -> ContextEntity: + """Build an entity representing the HuBMAP organization.""" hubmap_org = crate.add( ContextEntity( crate, @@ -188,9 +186,7 @@ def build_hubmap_org_entity(crate: ROCrate) -> ContextEntity: def build_primary_prov(ds_entity: WrappedEntity, crate: ROCrate) -> ContextEntity: - """ - This is largely a dummy routine for now. - """ + """Construct minimal dummy provenance.""" hubmap_org = build_hubmap_org_entity(crate) agent = crate.add( SoftwareApplication( @@ -224,6 +220,7 @@ def build_primary_prov(ds_entity: WrappedEntity, crate: ROCrate) -> ContextEntit def build_cwl_entity(crate: ROCrate) -> ContextEntity: + """Build an entity representing the CWL language.""" props = { "@id": "https://w3id.org/workflowhub/workflow-ro-crate#cwl", "@type": "ComputerLanguage", @@ -238,6 +235,7 @@ def build_cwl_entity(crate: ROCrate) -> ContextEntity: def build_step_entity( step: dict, idx: int, cwl_entity: ContextEntity, crate: ROCrate ) -> ContextEntity: + """Build an entity representing one step of the provenance chain.""" pos = idx + 1 id_str = f"#step_{pos}" hash = step["commit"] @@ -260,6 +258,7 @@ def build_step_entity( def build_derived_prov(ds_entity: WrappedEntity, crate: ROCrate) -> ContextEntity: + """Build a set of entities representing the provenance by which a derived dataset is produced from one or more primary datasets.""" prov_chain = ds_entity.walk_ancestors(lambda d: d["entity_type"] == "Dataset") assert len(prov_chain) == 1 assert len(prov_chain[0]) == 3 @@ -365,9 +364,7 @@ def build_derived_prov(ds_entity: WrappedEntity, crate: ROCrate) -> ContextEntit def build_profiles(crate: ROCrate) -> tuple: - """ - Build ContextElements for several profiles needed to describe a workflow. - """ + """Build ContextElements for several profiles needed to describe a workflow.""" base_crate_ctx_id = f"https://w3id.org/ro/crate/{crate.version}" crate_profile = crate.add( ContextEntity( @@ -417,6 +414,7 @@ def build_profiles(crate: ROCrate) -> tuple: def main() -> None: + """Parse command line and carry out construction of the RO-Crate and Croissant files.""" parser = argparse.ArgumentParser() parser.add_argument("target_id", help="Build an RO-Crate for this dataset") parser.add_argument( diff --git a/src/crate_builder_script/croissant_wrapper.py b/src/crate_builder_script/croissant_wrapper.py index e972460..6257eb9 100644 --- a/src/crate_builder_script/croissant_wrapper.py +++ b/src/crate_builder_script/croissant_wrapper.py @@ -1,3 +1,4 @@ +"""Provides functions to build the Croissant file.""" import json import logging from pprint import pformat @@ -146,6 +147,8 @@ def build_embedded_provenance( entity: WrappedEntity, md: dict | None, descendants: list | None = None ) -> dict: """ + Build embedded provenance for the given entity. + PROCESSED subject: wasGeneratedBy its own pipeline; wasDerivedFrom the raw parent (which carries the acquisition activity + specimen chain). RAW subject: wasGeneratedBy acquisition; wasDerivedFrom specimen chain; + a light forward pointer to processed versions. @@ -188,8 +191,11 @@ def build_embedded_provenance( class CroissantWrapper: + """Wraps Croissant generation functionality.""" + @classmethod def test(cls, entity_dict: dict) -> None: + """Provide a test of utility methods.""" LOGGER.info( "Testing CroissantWrapper:\n%s", pformat( @@ -202,6 +208,7 @@ def test(cls, entity_dict: dict) -> None: ) def __init__(self, name: str, description: str): + """Construct a CroissantWrapper instance.""" self.name = name self.description = description self.file_objects = [] @@ -212,6 +219,7 @@ def __init__(self, name: str, description: str): self.version = None def add_file(self, ds_uuid: str, file_info: dict, file_blk: dict | None) -> None: + """Define and describe a single file in the dataset.""" args = { "id": file_info["rel_path"], "name": file_info["rel_path"], @@ -225,7 +233,7 @@ def add_file(self, ds_uuid: str, file_info: dict, file_blk: dict | None) -> None LOGGER.warning(f"Unknown EDAM format {edam} for {pformat(file_info)}") args["encoding_formats"] = ["application/octet-stream"] else: - args[encoding_formats] = ["application/octet-stream"] + args["encoding_formats"] = ["application/octet-stream"] if file_blk: args["sha256"] = file_blk["sha256_checksum"] self.file_objects.append(mlc.FileObject(**args)) @@ -234,6 +242,7 @@ def add_file(self, ds_uuid: str, file_info: dict, file_blk: dict | None) -> None # self.record_sets.append(record_set) def write(self, croissant_filename: str): + """Write the Croissant file.""" args = { "id": "croissant-spec", "name": self.name, diff --git a/src/crate_builder_script/extractors.py b/src/crate_builder_script/extractors.py index e2393a3..b3da4ef 100644 --- a/src/crate_builder_script/extractors.py +++ b/src/crate_builder_script/extractors.py @@ -1,3 +1,5 @@ +"""Define utilities to extract relevant subsets of entity data.""" + import logging from collections.abc import Callable from pprint import pformat @@ -12,6 +14,8 @@ def walk_ancestors( entity: dict, continue_test: Callable[[dict], bool] = lambda ent: True ) -> list[tuple]: """ + Extract the chain of ancestors for a dataset. + Given an entity dictionary, return a list of tuples. Each tuple has the form: (hubmap_id entity_dict list-of-ancestors) @@ -63,6 +67,7 @@ def walk_ancestors( def listify(ancestor_chain: list, omit_test: Callable[[dict], bool]) -> list: + """Transform the output of walk_ancestors() to a list.""" assert len(ancestor_chain) == 1, "listify must start on a 1-tuple chain" hubmap_id, entity_dict, ancestors = ancestor_chain[0] rslt = [] @@ -75,7 +80,11 @@ def listify(ancestor_chain: list, omit_test: Callable[[dict], bool]) -> list: def is_processed(entity: dict) -> bool: - """Raw vs processed: `creation_action` is 'Create Dataset Activity' vs 'Central Process'.""" + """ + Distinguished processed datasets from raw (primary) datasets. + + Raw vs processed: `creation_action` is 'Create Dataset Activity' vs 'Central Process'. + """ return "process" in (entity.get("creation_action") or "").lower() @@ -84,6 +93,7 @@ def _own_dag(entity: dict) -> list: def pipeline_steps(entity: dict) -> list[dict]: + """Return a list of dicts describing the steps by which a derived dataset is created.""" dag_list = _own_dag(entity) steps, seen = [], set() for s in dag_list or []: @@ -100,22 +110,30 @@ def pipeline_steps(entity: dict) -> list[dict]: class WrappedEntity: + """Provide a convenient wrapper for entity information.""" + def __init__(self, entity: dict): + """Construct a WrappedEntity.""" self._entity = entity def get(self, key: Any, default=None) -> Any: + """Pass references to the get method to the internal dict.""" return self._entity.get(key, default) def __getitem__(self, key: Any) -> Any: + """Implement square brackets for the wrapper class.""" return self._entity[key] def __contains__(self, key: Any) -> bool: + """Implement 'in' for the wrapper class.""" return key in self._entity def walk_ancestors( self, continue_test: Callable[[dict], bool] = lambda ent: True ) -> list[tuple]: """ + Walk the ancestors of the entity. + Return a list of tuples of the form: (hubmap_id entity_dict list-of-ancestors) where list-of-ancestors is None or a list of tuples of the same form. @@ -130,18 +148,20 @@ def list_ancestors( continue_test: Callable[[dict], bool] = lambda ent: True, omit_test: Callable[[dict], bool] = lambda end: False, ) -> list: - """Returns ancestor information in an expanded, non-recursive list""" + """Return ancestor information in an expanded, non-recursive list.""" return listify(self.walk_ancestors(continue_test), omit_test) @property def is_processed(self): - """Is this a processed dataset, as opposed to a raw (primary) dataset?""" + """True if this a processed dataset, as opposed to a raw (primary) dataset.""" return is_processed(self._entity) def pipeline_steps(self) -> list[dict]: + """Return the pipeline steps of the wrapped entity.""" return pipeline_steps(self._entity) def count_versions(self) -> int: + """Return a version number for the wrapped entity.""" if "previous_revision_uuid" in self: prev_ent = WrappedEntity(fetch_entity_info(self["previous_revision_uuid"])) return prev_ent.count_versions() + 1 From 743d9eb276d22bd3956ec9210fbb08195d24e228 Mon Sep 17 00:00:00 2001 From: Joel Welling Date: Thu, 3 Sep 2026 15:56:50 -0400 Subject: [PATCH 33/36] black --- .../build_crate_from_dataset.py | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/src/crate_builder_script/build_crate_from_dataset.py b/src/crate_builder_script/build_crate_from_dataset.py index 1d00bd3..1ac1b49 100644 --- a/src/crate_builder_script/build_crate_from_dataset.py +++ b/src/crate_builder_script/build_crate_from_dataset.py @@ -4,16 +4,26 @@ import os from collections import defaultdict from datetime import datetime, timezone + # from pprint import pformat, pprint from tempfile import TemporaryDirectory from typing import List -from api_calls import (HUBMAP_ORG_ENTITY, asset_url, fetch_entity_info, - fetch_uuid_files_info) +from api_calls import ( + HUBMAP_ORG_ENTITY, + asset_url, + fetch_entity_info, + fetch_uuid_files_info, +) from croissant_wrapper import CroissantWrapper from extractors import WrappedEntity -from rocrate.model import (ComputerLanguage, ContextEntity, Dataset, Person, - SoftwareApplication) +from rocrate.model import ( + ComputerLanguage, + ContextEntity, + Dataset, + Person, + SoftwareApplication, +) from rocrate.rocrate import ROCrate logging.basicConfig( From de7bb48629a2f94445200af3acf30badf6d47915 Mon Sep 17 00:00:00 2001 From: Joel Welling Date: Thu, 3 Sep 2026 16:50:21 -0400 Subject: [PATCH 34/36] add pyproject.toml --- pyproject.toml | 4 ++++ 1 file changed, 4 insertions(+) create mode 100644 pyproject.toml diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..17b0c37 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,4 @@ +[tool.isort] +profile = "black" +multi_line_output = 3 + From 8e84c2d7f496511a8f407bf3e740f56829b045b7 Mon Sep 17 00:00:00 2001 From: Joel Welling Date: Thu, 3 Sep 2026 17:26:18 -0400 Subject: [PATCH 35/36] trying to fix python 3.13 pandas build error --- src/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/requirements.txt b/src/requirements.txt index d833963..f018eba 100644 --- a/src/requirements.txt +++ b/src/requirements.txt @@ -1,5 +1,5 @@ jsonschema==4.23.0 -pandas==2.2.2 +pandas>=2.2.2 PyYAML==6.0.2 requests==2.32.3 rocrate>=0.15.0 From 9cdb3413e8c7f7c5182bdb4d2b6d68b82f52e9aa Mon Sep 17 00:00:00 2001 From: Joel Welling Date: Fri, 4 Sep 2026 20:45:40 -0400 Subject: [PATCH 36/36] clean up get() defaults --- src/crate_builder_script/croissant_wrapper.py | 4 ++-- src/crate_builder_script/extractors.py | 12 ++++++------ 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/src/crate_builder_script/croissant_wrapper.py b/src/crate_builder_script/croissant_wrapper.py index 6257eb9..19dde75 100644 --- a/src/crate_builder_script/croissant_wrapper.py +++ b/src/crate_builder_script/croissant_wrapper.py @@ -95,7 +95,7 @@ def _specimen_chain(entity: WrappedEntity) -> dict: def node(anc): n = { "@type": "prov:Entity", - "@id": HUBMAP + (anc.get("hubmap_id") or ""), + "@id": HUBMAP + anc.get("hubmap_id", ""), "schema:name": anc.get("hubmap_id"), "hubmap:entityType": anc.get("entity_type"), "hubmap:sampleCategory": anc.get("sample_category"), @@ -181,7 +181,7 @@ def build_embedded_provenance( provo["hubmap:hasProcessedDataset"] = [ { "@type": "prov:Entity", - "@id": HUBMAP + (d.get("hubmap_id") or ""), + "@id": HUBMAP + d.get("hubmap_id", ""), "schema:name": d.get("hubmap_id"), "hubmap:datasetType": d.get("dataset_type"), } diff --git a/src/crate_builder_script/extractors.py b/src/crate_builder_script/extractors.py index b3da4ef..9a4e13e 100644 --- a/src/crate_builder_script/extractors.py +++ b/src/crate_builder_script/extractors.py @@ -85,11 +85,11 @@ def is_processed(entity: dict) -> bool: Raw vs processed: `creation_action` is 'Create Dataset Activity' vs 'Central Process'. """ - return "process" in (entity.get("creation_action") or "").lower() + return "process" in entity.get("creation_action", "").lower() def _own_dag(entity: dict) -> list: - return (entity.get("ingest_metadata") or {}).get("dag_provenance_list", []) + return entity.get("ingest_metadata", {}).get("dag_provenance_list", []) def pipeline_steps(entity: dict) -> list[dict]: @@ -97,10 +97,10 @@ def pipeline_steps(entity: dict) -> list[dict]: dag_list = _own_dag(entity) steps, seen = [], set() for s in dag_list or []: - repo = (s.get("origin") or "").strip().replace(".git", "") - name = repo.rsplit("/", 1)[-1] if repo else (s.get("name") or "") - commit = (s.get("hash") or "")[:7] - cwl = s.get("name") or "" + repo = s.get("origin", "").strip().replace(".git", "") + name = repo.rsplit("/", 1)[-1] if repo else s.get("name", "") + commit = s.get("hash", "")[:7] + cwl = s.get("name", "") key = (name, commit, cwl) if not name or key in seen: continue