diff --git a/.github/ISSUE_TEMPLATE/new_dataset.yaml b/.github/ISSUE_TEMPLATE/new_dataset.yaml index 12796bc..7f93033 100644 --- a/.github/ISSUE_TEMPLATE/new_dataset.yaml +++ b/.github/ISSUE_TEMPLATE/new_dataset.yaml @@ -66,6 +66,14 @@ body: placeholder: ex. md5:df9376d5c020a23f0f7f51cfe79f303f validations: required: false + - type: input + id: size_bytes + attributes: + label: --Size (bytes)-- + description: The size of the file in bytes. Optional. Filled in automatically when it is left blank by asking the server for the file's `Content-Length`; the Add Dataset form's file picker fills it in from your local copy. + placeholder: ex. 1104287335 + validations: + required: false - type: textarea id: description attributes: diff --git a/.github/scripts/issue_to_yaml.py b/.github/scripts/issue_to_yaml.py index 6d726f7..b2ba6df 100644 --- a/.github/scripts/issue_to_yaml.py +++ b/.github/scripts/issue_to_yaml.py @@ -7,9 +7,9 @@ the test suite and ``emdatabase.new_dataset`` run - so a malformed issue fails here rather than in the pull request the workflow opens. -The form's checksum is optional, and the size is only what a HEAD request said, -so ``emdatabase.new_dataset.fill_download_fields`` downloads the file for -whichever of the two the issue left blank before any of that. +The form's checksum and size are both optional - the size falls back to what a +HEAD request says - so ``emdatabase.new_dataset.fill_download_fields`` downloads +the file for whichever of the two is still missing before any of that. """ import re @@ -37,6 +37,7 @@ "URL", "File Name", "Checksum", + "Size (bytes)", "Description", "Detector Manufacturer", "Detector Model", @@ -111,6 +112,9 @@ def build_yaml(data): filename = data["File Name"] or filename if not filename: sys.exit(f"{data['URL']!r} does not end in a file name; fill in --File Name--") + # The Add Dataset form's file picker fills the size in from the local copy; + # without it the server is asked for the file's Content-Length. + size = data["Size (bytes)"].replace(",", "").replace("_", "").strip() entry = { "description": data["Description"], @@ -118,7 +122,7 @@ def build_yaml(data): "url": link, "checksum": data["Checksum"], "file": filename, - "size_bytes": content_length(url), + "size_bytes": int(size) if size.isdigit() else content_length(url), "detector_manufacturer": data["Detector Manufacturer"], "detector": data["Detector Model"], "microscope_vendor": data["Microscope Vendor"], diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 6096665..4d18d84 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -1,11 +1,14 @@ # Contributing -A dataset is one YAML file in `emdatabase/index/`. To add one, either fill in the -[new-dataset issue form](https://github.com/electronmicroscopy/emdatabase/issues/new?template=new_dataset.yaml), -which opens the pull request for you, or the -[Add Dataset form](https://electronmicroscopy.github.io/emdatabase/add_dataset.html), -which takes one download link - a Google Drive share link included - and fills in the -file name, size and md5 from your local copy of the file. Or run +A dataset is one YAML file in `emdatabase/index/`. To add one, fill in the +[new-dataset issue form](https://github.com/electronmicroscopy/emdatabase/issues/new?template=new_dataset.yaml) +and an action turns it into that file and opens the pull request for you. The +[Add Dataset form](https://electronmicroscopy.github.io/emdatabase/add_dataset.html) +is the easier way in: it takes one download link - a Google Drive share link included - +fills in the file name, size and md5 from your local copy of the file, checks +everything, and opens the issue form with all of it already filled in. + +If you would rather stay in a terminal, run ```bash python -m emdatabase.new_dataset https://zenodo.org/records//files/ @@ -14,8 +17,8 @@ python -m emdatabase.new_dataset https://zenodo.org/records//files/. diff --git a/README.md b/README.md index 56f4d6d..a6d498d 100644 --- a/README.md +++ b/README.md @@ -191,20 +191,23 @@ vendors and detector manufacturers already in use - a new one is fine, but a nam to one already on the list fails CI as a misspelling. A technique close to one in `techniques.yaml` fails the same way. -Open an issue with the [new dataset template](https://github.com/electronmicroscopy/emdatabase/issues/new?template=new_dataset.yaml), -fill in the [Add Dataset form](https://electronmicroscopy.github.io/emdatabase/add_dataset.html), -or run `python -m emdatabase.new_dataset `, which fetches the checksum and size, -prompts for the rest and writes the file for you to open a pull request with. See +Submissions go through an issue. Fill in the +[Add Dataset form](https://electronmicroscopy.github.io/emdatabase/add_dataset.html), +which checks the fields and opens the +[new dataset issue](https://github.com/electronmicroscopy/emdatabase/issues/new?template=new_dataset.yaml) +prefilled from them, and an action writes the file and opens the pull request. Or run +`python -m emdatabase.new_dataset `, which fetches the checksum and size, prompts +for the rest and writes the file for you to open a pull request with. See [CONTRIBUTING.md](CONTRIBUTING.md). -All three take one download link and split it into `source`, `file` and, when the file -is not served at `source/file`, `url`. A Google Drive share link - what the share -button copies - is rewritten to the `uc?export=download&id=` link that serves the -file. The form also has a local file picker: point it at the copy on your machine and -it fills in the file name, size and md5, hashing the file in the browser without -uploading it. - -Neither web route needs the checksum or the size. A pull request carrying an entry -that is missing either one has the file downloaded on GitHub and the fields filled in -and pushed back to the branch; a pull request from a fork, whose branch cannot be -pushed to, fails with the values to paste in instead. +Both take one download link and split it into `source`, `file` and, when the file is +not served at `source/file`, `url`. A Google Drive share link - what the share button +copies - is rewritten to the `uc?export=download&id=` link that serves the file. +The form also has a local file picker: point it at the copy on your machine and it +fills in the file name, size and md5, hashing the file in the browser without uploading +it. + +Neither route needs the checksum or the size. An entry that is missing either one has +the file downloaded on GitHub and the fields filled in and pushed back to the branch; a +pull request from a fork, whose branch cannot be pushed to, fails with the values to +paste in instead. diff --git a/docs/source/_build_docs.py b/docs/source/_build_docs.py index 4e01297..046c9ad 100644 --- a/docs/source/_build_docs.py +++ b/docs/source/_build_docs.py @@ -789,11 +789,6 @@ def generate_html_table(datasets_by_technique): } .btn-primary:disabled { opacity: 0.45; cursor: not-allowed; filter: grayscale(0.3); } .btn-primary:not(:disabled):hover { filter: brightness(1.06); } -.btn-secondary { - text-align: center; font-size: 12px; font-weight: 600; color: var(--emdb-subtext); - border: 1px solid var(--emdb-surface1); border-radius: 8px; padding: 9px 14px; -} -.btn-secondary:hover { color: var(--emdb-text); border-color: var(--emdb-blue); text-decoration: none; } .copy-mini { cursor: pointer; font-size: 11px; font-weight: 600; color: var(--emdb-blue); background: transparent; border: 1px solid var(--emdb-surface1); border-radius: 6px; padding: 2px 9px; @@ -1008,9 +1003,11 @@ def generate_weights_html() -> str: "for a very large file, pick the local file above instead." ) -# Owner/repo the prefilled "create new file" PR link targets. -_REPO = "electronmicroscopy/emdatabase" -_BRANCH = "main" +# The new-dataset issue form. The page's submit button appends the field values +# to this as query parameters, which is how GitHub prefills an issue form. +_ISSUE_URL = ( + "https://github.com/electronmicroscopy/emdatabase/issues/new?template=new_dataset.yaml" +) # md5 in the browser, for the Add Dataset page's local-file picker - the one # external resource any generated page loads. If it does not load the picker @@ -1170,7 +1167,7 @@ def _author_row_html(): def generate_add_dataset_html() -> str: - """The Add Dataset page: a schema-driven form that opens a prefilled PR.""" + """The Add Dataset page: a schema-driven form that opens a prefilled issue.""" fields = ( _text_field( "f-name", @@ -1290,24 +1287,27 @@ def generate_add_dataset_html() -> str: ) ) - issue_url = "https://github.com/" + _REPO + "/issues/new?template=new_dataset.yaml" - body = ( '
' '
' '

Add a Dataset

' - "

Fill in the metadata; the YAML builds live on the right. " - "“Open a Pull Request” sends you to GitHub with the new file " - "pre-filled — commit it to a branch there and GitHub opens the PR.

" - "

Pick the file from your machine to fill in its name, size and md5. " - "From a terminal, python -m emdatabase.new_dataset <url> " - "does the same from the link; see " - 'Contributing a Dataset.

' + "

Submissions go through an issue. Fill in the metadata below and " + "“Submit as an issue” opens the new-dataset issue form on GitHub " + "with these fields already filled in; an action turns the issue into the " + "entry, fills in whatever is missing and opens the pull request.

" + "

The form checks each field as you type, and picking the file from your " + "machine fills in its name, size and md5 — it is hashed in the browser " + "and nothing is uploaded. If you would rather stay in a terminal, " + "python -m emdatabase.new_dataset <url> asks the same " + 'questions there; see Contributing a Dataset.

' "
" '
' '
' '
' + fields + "
" '
Authors
' + '
The issue carries the ' + "first author only; the rest can be added to the YAML on the pull " + "request.
" '
' + _author_row_html() + "
" '' '
Entry
' @@ -1321,27 +1321,27 @@ def generate_add_dataset_html() -> str: "
" "" '" "" "
" ) - js = _ADD_DATASET_JS.replace("__REPO__", _REPO).replace("__BRANCH__", _BRANCH) + js = _ADD_DATASET_JS.replace("__ISSUE_URL__", _ISSUE_URL) scripts = _SPARK_MD5_SRC + "\n" return _app_page( "Add Dataset · EM-Database", @@ -1498,12 +1498,93 @@ def generate_add_dataset_html() -> str: return lines.join("\n") + "\n"; } +// The issue form's field ids, keyed by the field names above. GitHub prefills an +// issue form from query parameters named after each field's `id`, which it does +// for `input` and `textarea` fields only - `technique` is a checkboxes field and +// `kind` a dropdown, so those are set on the issue itself. `kind` is sent anyway, +// against the day GitHub honours it. +var EMDB_ISSUE_IDS = [ + ["name", "dataset_name"], + ["link", "url"], + ["file", "file_name"], + ["checksum", "checksum"], + ["size_bytes", "size_bytes"], + ["description", "description"], + ["detector_manufacturer", "detector_manufacturer"], + ["detector", "detector_model"], + ["microscope_vendor", "microscope_vendor"], + ["microscope_model", "microscope_model"], + ["camera_length", "camera_length"], + ["voltage", "accelerating_voltage"], + ["license", "license"], + ["doi", "doi"], + ["tags", "tags"], + ["kind", "kind"], + ["version_date", "version_date"], + ["model_class", "model_class"], + ["model_framework", "model_framework"], + ["model_quantem", "model_quantem"] +]; + +// A prefilled URL over ~8k characters is refused as `414 URI Too Long`, so the +// description is cut back to fit rather than the whole submission failing. +var EMDB_ISSUE_URL_MAX = 8000; + +// `{issue field id: value}` for one set of form values. The issue form asks for +// one author, so the first filled-in row is the one it carries. +function emdbIssueFields(fields) { + var out = {}; + EMDB_ISSUE_IDS.forEach(function (pair) { + var v = fields[pair[0]]; + if (Array.isArray(v)) v = v.join(", "); + v = v == null ? "" : String(v).trim(); + if (v) out[pair[1]] = v; + }); + var author = (fields.authors || []).filter(function (a) { return a && a.name; })[0]; + if (author) { + out.name = String(author.name).trim(); + if (author.aff) out.affiliation = String(author.aff).trim(); + if (author.orcid) out.orcid = String(author.orcid).trim(); + } + return out; +} + +// `{url, trimmed}` - the prefilled issue link, and how many characters of the +// description had to be left out of it. +function emdbIssueUrl(base, fields) { + var params = emdbIssueFields(fields); + var description = params.description || ""; + function build(desc) { + var parts = []; + Object.keys(params).forEach(function (id) { + var value = id === "description" ? desc : params[id]; + if (value) parts.push(encodeURIComponent(id) + "=" + encodeURIComponent(value)); + }); + return base + (parts.length ? "&" + parts.join("&") : ""); + } + if (build(description).length <= EMDB_ISSUE_URL_MAX) { + return { url: build(description), trimmed: 0 }; + } + var kept = 0, most = description.length; + while (kept < most) { + var mid = Math.ceil((kept + most) / 2); + if (build(description.slice(0, mid) + " …").length <= EMDB_ISSUE_URL_MAX) kept = mid; + else most = mid - 1; + } + return { + url: build(kept ? description.slice(0, kept) + " …" : ""), + trimmed: description.length - kept + }; +} + if (typeof module !== "undefined" && module.exports) { module.exports = { emdbBuildYaml: emdbBuildYaml, emdbEntryName: emdbEntryName, emdbNormalizeUrl: emdbNormalizeUrl, - emdbSplitUrl: emdbSplitUrl + emdbSplitUrl: emdbSplitUrl, + emdbIssueFields: emdbIssueFields, + emdbIssueUrl: emdbIssueUrl }; } """ @@ -1511,10 +1592,11 @@ def generate_add_dataset_html() -> str: _ADD_DATASET_JS = r""" (function () { - var REPO = "__REPO__", BRANCH = "__BRANCH__"; + var ISSUE_URL = "__ISSUE_URL__"; var form = document.getElementById("ds-form"); var preview = document.querySelector("#yaml-preview code"); - var submitPr = document.getElementById("submit-pr"); + var submitIssue = document.getElementById("submit-issue"); + var trimNote = document.getElementById("issue-trim"); var copyBtn = document.getElementById("copy-yaml"); var addAuthor = document.getElementById("add-author"); var authorsBox = document.getElementById("authors"); @@ -1619,8 +1701,15 @@ def generate_add_dataset_html() -> str: function refresh() { modelGroup.hidden = val("f-kind") !== "weights"; - preview.textContent = emdbBuildYaml(collect()); - submitPr.disabled = !validate(); + var fields = collect(); + preview.textContent = emdbBuildYaml(fields); + submitIssue.disabled = !validate(); + var trimmed = emdbIssueUrl(ISSUE_URL, fields).trimmed; + trimNote.hidden = !trimmed; + trimNote.textContent = trimmed + ? "The description is too long for a link, so the issue opens with its last " + + trimmed + " characters cut off - paste them back in before you submit it." + : ""; } function copyText(text, btn) { @@ -1699,13 +1788,9 @@ def generate_add_dataset_html() -> str: copyBtn.addEventListener("click", function () { copyText(emdbBuildYaml(collect()), copyBtn); }); - submitPr.addEventListener("click", function () { + submitIssue.addEventListener("click", function () { if (!validate()) { refresh(); return; } - var name = emdbEntryName(val("f-name")) || "Dataset"; - var url = "https://github.com/" + REPO + "/new/" + BRANCH - + "?filename=" + encodeURIComponent("emdatabase/index/" + name + ".yaml") - + "&value=" + encodeURIComponent(emdbBuildYaml(collect())); - window.open(url, "_blank", "noopener"); + window.open(emdbIssueUrl(ISSUE_URL, collect()).url, "_blank", "noopener"); }); refresh(); diff --git a/docs/source/contributing.rst b/docs/source/contributing.rst index 954ec1c..376e605 100644 --- a/docs/source/contributing.rst +++ b/docs/source/contributing.rst @@ -6,18 +6,21 @@ Contributing a Dataset A dataset is one YAML file in ``emdatabase/index/``, validated against ``emdatabase/index/json-schema.json``. Adding one means adding that file. -Three routes ------------- +Two routes +---------- Fill in the `new-dataset issue form `_ -and an action turns it into the YAML file and opens a pull request for you. Or -fill in the :doc:`Add Dataset ` form, which builds the YAML in the -browser and sends you to GitHub with the file pre-filled. Or run the CLI below, -which writes the file locally and leaves the pull request to you. All three end -in the same place, and all three run the same validator. Neither web route has -to be given the checksum or the size: a pull request carrying an entry that is -missing either one has the file downloaded and the fields filled in for it. +and an action turns it into the YAML file and opens a pull request for you. The +:doc:`Add Dataset ` form is the front door to that: it carries +every field the schema has, checks each one, computes the checksum from your +local copy of the file, and then opens the issue form with all of it filled in. + +Or run the CLI below, which asks the same questions in a terminal and writes the +file locally, leaving the pull request to you. Both routes end in the same file +and both run the same validator. Neither has to be given the checksum or the +size: an entry missing either one has the file downloaded and the fields filled +in for it. Techniques ---------- @@ -53,10 +56,19 @@ your machine, so the checksum need not be computed by hand. The file is read in the browser and nothing is uploaded; a multi-GB file is hashed a chunk at a time, with progress under the picker. The three fields it fills stay editable. -**Checksum** and **Size (bytes)** may be left blank, in which case the pull -request downloads the file and fills them in. The picker is the quicker route -for a large file, and a pull request from a fork has to use it, because a fork's -branch cannot be pushed to. +**Submit as an issue** opens the new-dataset issue form with everything you +typed already in it. GitHub prefills text fields only, so **Technique** has to +be ticked on the issue itself, and **Kind** set to ``weights`` there for a model +checkpoint. A description too long to fit in a link is cut short, with a note on +the page saying how much has to be pasted back in. + +The panel below the button is the entry the issue will produce. Nothing has to +be done with it - it is there for reading, and for the **Copy** button, since +the CLI writes the same file. + +**Checksum** and **Size (bytes)** may be left blank, in which case the file is +downloaded and the fields filled in. The picker is the quicker route for a large +file. Using the CLI ------------- @@ -160,12 +172,14 @@ one warns and asks for it to be added. A weights entry without an ML task, and a dataset with one, fail as well. ``fill_download_fields.yml`` runs on every pull request that touches -``emdatabase/index/``. It downloads the file behind each changed entry that is -missing its ``checksum`` or ``size_bytes`` - a weights family's ``latest`` and -each dated version on their own links - fills the fields in and pushes the -result back to the branch, which is how an entry from the web form or the issue -form ends up complete. A fork's branch cannot be pushed to, so a pull request -from one fails instead and prints the values to paste in. +``emdatabase/index/``, which is to say on hand-edited and CLI-written index +files. It downloads the file behind each changed entry that is missing its +``checksum`` or ``size_bytes`` - a weights family's ``latest`` and each dated +version on their own links - fills the fields in and pushes the result back to +the branch. A fork's branch cannot be pushed to, so a pull request from one +fails instead and prints the values to paste in. An entry coming in through the +issue form is filled in the same way before its pull request is opened, so it +arrives complete. ``check_sources.yml`` runs weekly and asks each source server whether the file is still there and still the size the entry claims. @@ -218,6 +232,6 @@ GitHub archival is only for links that move in place, Google Drive among them. Google Drive works for a small file, as a ``https://drive.google.com/uc?export=download&id=`` link written to the entry's ``url``; above about 100 MB Drive answers with a virus-scan page -instead of the file, and the entry will not download. All three routes take the +instead of the file, and the entry will not download. Both routes take the share link as well and rewrite it to that form, and fill in ``url``, ``source`` and ``file`` themselves. diff --git a/emdatabase/tests/test_forms.py b/emdatabase/tests/test_forms.py index 10e45f2..b0aa783 100644 --- a/emdatabase/tests/test_forms.py +++ b/emdatabase/tests/test_forms.py @@ -24,6 +24,7 @@ import sys from pathlib import Path from typing import Any +from urllib.parse import parse_qsl, urlsplit import pytest import yaml @@ -104,6 +105,43 @@ def run(fields): return run +@pytest.fixture +def run_issue_url(build_docs, tmp_path): + """Run the form's issue-link builder under node. + + Returns ``(params, built)`` - the link's query parameters, decoded and with + the template name dropped, and the ``{url, trimmed, max}`` the builder gave. + """ + node = shutil.which("node") + if node is None: + pytest.skip("node is not installed") + script = tmp_path / "issue_url.js" + script.write_text( + build_docs.ADD_DATASET_YAML_JS + + '\nvar fields = JSON.parse(require("fs").readFileSync(0, "utf8"));\n' + + "process.stdout.write(JSON.stringify(Object.assign(emdbIssueUrl(" + + json.dumps(build_docs._ISSUE_URL) + + ", fields), {max: EMDB_ISSUE_URL_MAX})));\n", + encoding="utf-8", + ) + + def run(fields): + result = subprocess.run( + [node, str(script)], + input=json.dumps(fields), + capture_output=True, + text=True, + check=True, + ) + built = json.loads(result.stdout) + assert built["url"].startswith(build_docs._ISSUE_URL) + params = dict(parse_qsl(urlsplit(built["url"]).query)) + assert params.pop("template") == "new_dataset.yaml" + return params, built + + return run + + @pytest.fixture def run_split(build_docs, tmp_path): """Run the form's link splitter under node; return ``(source, file, url)``.""" @@ -288,6 +326,100 @@ def test_form_takes_a_drive_share_link(run_form): assert entry["file"] == "example.zspy" +# What the issue form's field ids are called on the docs form. The submit link +# is only worth having if the two agree, so the mapping is written out here +# rather than read back off the page. +ISSUE_IDS = { + "dataset_name": "DemoNet", + "name": "Jane Doe", + "affiliation": "University of Somewhere", + "orcid": "0000-0002-1825-0097", + "url": DRIVE_LINK, + "file_name": "DemoNet.pt", + "checksum": "md5:df9376d5c020a23f0f7f51cfe79f303f", + "size_bytes": "1104287335", + "description": "Trained weights for the peak-finding U-Net.", + "detector_manufacturer": "Direct Electron", + "detector_model": "CeleritasXS", + "microscope_vendor": "Thermo Fisher Scientific", + "microscope_model": "Gen 1 Titan", + "camera_length": "100 mm", + "accelerating_voltage": "200 kV", + "license": "CC-BY-4.0", + "doi": "10.5281/zenodo.15490547", + "tags": "Nanocrystals, Orientation Mapping", + "kind": "weights", + "version_date": VERSION_DATE, + "model_class": "quantem.core.ml.CNN2d", + "model_framework": "torch", + "model_quantem": ">=0.2,<0.3", +} + + +def test_form_submits_as_an_issue(build_docs): + """One button, and it goes to the issue form; nothing opens a pull request.""" + html = build_docs.generate_add_dataset_html() + assert 'id="submit-issue" class="btn-primary"' in html + assert build_docs._ISSUE_URL in html + assert build_docs._ISSUE_URL.endswith("/issues/new?template=new_dataset.yaml") + assert "/new/main" not in html and "filename=" not in html + assert "create new file" not in html + assert not hasattr(build_docs, "_REPO") and not hasattr(build_docs, "_BRANCH") + + +def test_form_shows_the_yaml_below_the_submit_button(build_docs): + """The entry is a preview of what the issue produces, not the way to submit it.""" + html = build_docs.generate_add_dataset_html() + assert html.index('id="submit-issue"') < html.index('id="copy-yaml"') + assert html.index('id="copy-yaml"') < html.index('id="yaml-preview"') + + +def test_form_issue_link_carries_every_field_the_template_has(run_issue_url): + """The link's query has to decode back to the values the YAML is built from.""" + params, built = run_issue_url(WEIGHTS_FIELDS) + assert params == ISSUE_IDS + assert built["trimmed"] == 0 + + +def test_form_issue_link_ids_are_the_ones_the_template_declares(run_issue_url): + """Every prefillable field on the template, and nothing GitHub will not prefill.""" + form = yaml.safe_load(ISSUE_FORM.read_text(encoding="utf-8")) + blocks = {b["id"]: b["type"] for b in form["body"] if b["type"] != "markdown"} + params, _built = run_issue_url(WEIGHTS_FIELDS) + assert set(params) <= set(blocks) + # GitHub prefills `input` and `textarea` only, so the technique tick boxes + # are left to be filled in on the issue itself. + assert blocks["technique"] == "checkboxes" + assert "technique" not in params + assert set(blocks) - set(params) == {"technique"} + + +def test_form_issue_link_leaves_out_what_the_form_was_not_given(run_issue_url): + """A blank field is absent from the query rather than sent empty.""" + params, _built = run_issue_url(dict(DATASET_FIELDS, doi="", authors=[], tags=[])) + assert "doi" not in params and "tags" not in params + assert "name" not in params and "affiliation" not in params + assert params["dataset_name"] == "MgONanoCrystals" + assert params["kind"] == "dataset" + + +def test_form_issue_link_trims_a_description_that_will_not_fit(run_issue_url): + """GitHub answers an over-long URL with 414, so the description gives way.""" + long_description = ("A 4D-STEM dataset of MgO nanocrystals. " * 400).strip() + params, built = run_issue_url(dict(DATASET_FIELDS, description=long_description)) + trimmed = built["trimmed"] + assert trimmed > 0 + assert len(built["url"]) <= built["max"] <= 8192 + assert len(params["description"]) + trimmed == len(long_description) + 2 + assert params["description"].endswith(" \u2026") + for key, value in ISSUE_IDS.items(): + if key in ("description", "dataset_name", "file_name", "kind", "version_date"): + continue + if key.startswith("model_"): + continue + assert params[key] == value + + def test_form_offers_a_local_file_picker(build_docs): """The picker fills in name, size and md5; the md5 comes from SparkMD5.""" html = build_docs.generate_add_dataset_html() @@ -544,6 +676,49 @@ def test_issue_drive_link_becomes_url_plus_file_name(parse): assert "model" not in entry +def test_issue_takes_the_size_the_form_measured(issue_to_yaml, monkeypatch): + """The picker fills the size in, so the server is not asked for it.""" + + def no_head(url): + raise AssertionError(f"asked the server about {url}") + + monkeypatch.setattr(issue_to_yaml, "content_length", no_head) + body = _issue_body( + **{ + "--Dataset Name--": "MgONanoCrystals", + "--URL--": "https://drive.google.com/uc?export=download&id=1inQ6DQ2zH40Ccd", + "--File Name--": "MgONanoCrystals.zspy", + "--Checksum--": "md5:df9376d5c020a23f0f7f51cfe79f303f", + "--Size (bytes)--": "1104287335", + "--Description--": "A 4D-STEM dataset of MgO nanocrystals.", + "--Dataset License--": "CC-BY-4.0", + "Technique": _ticked("4D-STEM"), + } + ) + document, name = issue_to_yaml.build_yaml(issue_to_yaml.parse_issue_body(body)) + assert validate_document(document) == [] + assert document[name]["size_bytes"] == 1104287335 + + +def test_issue_asks_the_server_when_the_size_is_not_a_number(issue_to_yaml, monkeypatch): + """A blank or unparseable answer falls back to the HEAD request.""" + monkeypatch.setattr(issue_to_yaml, "content_length", lambda url: 4242) + for answer in ("_No response_", "about 1 GB"): + body = _issue_body( + **{ + "--Dataset Name--": "MgONanoCrystals", + "--URL--": "https://zenodo.org/records/1/files/MgONanoCrystals.zspy", + "--Checksum--": "md5:df9376d5c020a23f0f7f51cfe79f303f", + "--Size (bytes)--": answer, + "--Description--": "A 4D-STEM dataset of MgO nanocrystals.", + "--Dataset License--": "CC-BY-4.0", + "Technique": _ticked("4D-STEM"), + } + ) + document, name = issue_to_yaml.build_yaml(issue_to_yaml.parse_issue_body(body)) + assert document[name]["size_bytes"] == 4242 + + FILE_BYTES = b"a small 4D-STEM dataset, allegedly" * 100 FILE_MD5 = f"md5:{hashlib.md5(FILE_BYTES).hexdigest()}"