Follow gg2rdf to https:// subject IRIs and document the store migration

README.md

@@ -158,8 +158,8 @@ DELETE {
 } WHERE {
   GRAPH <TARGET> {
     ?r ?p ?o .
-    FILTER( STRSTARTS(STR(?r), "http://taxon-name.plazi.org/id/")
-         || STRSTARTS(STR(?r), "http://taxon-concept.plazi.org/id/") )
+    FILTER( STRSTARTS(STR(?r), "https://taxon-name.plazi.org/id/")
+         || STRSTARTS(STR(?r), "https://taxon-concept.plazi.org/id/") )
     FILTER NOT EXISTS { ?referrer ?referrerP ?r . FILTER(?referrer != ?r) }
   }
 }
@@ -244,6 +244,76 @@ Catalogue of Life triples that point at plazi taxon names protect them from the
 sweep like any other referrer, and a patch that removes the last such link
 merely leaves an orphan for the next sweep to collect.
 
+## Migrating a store to https:// IRIs
+
+Since [plazi/gg2rdf#34] (issue [#33][plazi/gg2rdf#33]) every Plazi resource
+IRI is `https://`, matching the graph names this loader has always used. A store
+loaded before that switch holds `http://` subjects, and this loader — configured
+for `https://` — cannot find them: in `single-graph` mode the delete follows
+links from the *stored* treatment IRI, so an update to an old-style treatment
+would insert the new one next to the old rather than replace it. Rename the
+store first, then switch the loader; the two steps have to happen together.
+
+[plazi/gg2rdf#33]: https://github.com/plazi/gg2rdf/issues/33
+[plazi/gg2rdf#34]: https://github.com/plazi/gg2rdf/pull/34
+
+There is no need to replay every treatment file. One update rewrites every
+affected IRI in subject and object position, scoped to our graph and nothing
+else — other graphs in a shared store belong to other publishers:
+
+```sparql
+DELETE { GRAPH <TARGET> { ?s ?p ?o } }
+INSERT { GRAPH <TARGET> { ?s2 ?p ?o2 } }
+WHERE {
+  GRAPH <TARGET> { ?s ?p ?o }
+  BIND(STRSTARTS(STR(?s), "http://treatment.plazi.org/")
+    || STRSTARTS(STR(?s), "http://taxon-name.plazi.org/")
+    || STRSTARTS(STR(?s), "http://taxon-concept.plazi.org/")
+    || STRSTARTS(STR(?s), "http://publication.plazi.org/")
+    || STRSTARTS(STR(?s), "http://tb.plazi.org/") AS ?renameS)
+  BIND(isIRI(?o) && (STRSTARTS(STR(?o), "http://treatment.plazi.org/")
+    || STRSTARTS(STR(?o), "http://taxon-name.plazi.org/")
+    || STRSTARTS(STR(?o), "http://taxon-concept.plazi.org/")
+    || STRSTARTS(STR(?o), "http://publication.plazi.org/")
+    || STRSTARTS(STR(?o), "http://tb.plazi.org/")) AS ?renameO)
+  FILTER(?renameS || ?renameO)
+  BIND(IF(?renameS, IRI(CONCAT("https", SUBSTR(STR(?s), 5))), ?s) AS ?s2)
+  BIND(IF(?renameO, IRI(CONCAT("https", SUBSTR(STR(?o), 5))), ?o) AS ?o2)
+}
+```
+
+In `graph-per-file` mode replace the fixed `<TARGET>` with `?g` and add
+`FILTER(STRSTARTS(STR(?g), "https://treatment.plazi.org/id/"))` to the `WHERE`
+clause; the graph names themselves do not change. `COL_GRAPH` is left out on
+purpose: the Catalogue of Life data holds only `catalogueoflife.org` IRIs and
+never mentions a Plazi resource, so there is nothing in it to rename. Where the
+endpoint does not take SPARQL Update well, `DROP GRAPH` plus `LOAD` per
+treatment from regenerated files is equivalent, since each graph holds exactly
+one file.
+
+Predicates need no renaming: gg2rdf never puts an instance IRI in predicate
+position, and the vocabulary namespaces (`trt:` and friends) stay `http://` by
+design. Derived triples such as the taxomplete index move with their subjects.
+The statement is one transaction; if the endpoint runs out of heap, split it
+into one statement per host — five transactions, each bounded.
+
+The order of operations:
+
+1. Stop the loader.
+2. Run the rename on the store.
+3. Deploy this version of the loader.
+4. Start it again. It now works against the renamed store even while
+   `treatments-rdf` still holds `http://` files: a file modified after the
+   gg2rdf switch deletes and re-inserts `https://` subjects, which is
+   idempotent. Regenerating `treatments-rdf` so that a reload from scratch
+   matches the store is off the critical path.
+
+Triples in *other* graphs that point at `http://` Plazi IRIs are left alone,
+deliberately: rewriting other publishers' data is not ours to do, and the old
+IRIs still dereference. Only graph joins against our data break for them, which
+is the failure this change removes on our side — tell publishers who join
+against Plazi IRIs the date.
+
 ## Development
 
 ```sh

config/config.ts

@@ -33,6 +33,9 @@ import { type SparqlConfig } from "../src/sparql.ts";
  *
  * The uri namespaces below are not deployment settings: they have to match what
  * gg2rdf writes, and changing them would orphan everything already uploaded.
+ * Since plazi/gg2rdf#34 (issue #33) that is `https://` throughout, for graph names and
+ * subjects alike; a store loaded before the switch has to be renamed in place
+ * first (README, "Migrating a store to https:// IRIs").
  */
 
 function required(name: string) {
@@ -72,10 +75,10 @@ export const sparqlConfig: SparqlConfig = mode === "single-graph"
       ["insert-data", "load"],
       "insert-data",
     ),
-    // note the http, and note that these are not the graph names used by
-    // graph-per-file below: they are the subject uris gg2rdf writes
-    treatmentUriPrefix: "http://treatment.plazi.org/id",
-    materialCitationUriPrefix: "http://tb.plazi.org/GgServer/dwcaRecords",
+    // these are not the graph names used by graph-per-file below: they are
+    // the subject uris gg2rdf writes (the same prefix, but a different role)
+    treatmentUriPrefix: "https://treatment.plazi.org/id",
+    materialCitationUriPrefix: "https://tb.plazi.org/GgServer/dwcaRecords",
   }
   : {
     mode,

src/col_test.ts

@@ -30,7 +30,7 @@ const NEW = "2026-09-25";
 
 /** A treatment's taxon name, which the Catalogue of Life data points at. */
 const foreign = `INSERT DATA { GRAPH <${GRAPH}> {
-  <http://taxon-name.plazi.org/id/Animalia/Saigona> a <http://filteredpush.org/ontologies/oa/dwcFP#TaxonName> .
+  <https://taxon-name.plazi.org/id/Animalia/Saigona> a <http://filteredpush.org/ontologies/oa/dwcFP#TaxonName> .
 } }`;
 
 const sorted = (lines: string[]) => [...lines].sort();
@@ -39,7 +39,7 @@ const sorted = (lines: string[]) => [...lines].sort();
 const oldSnapshot = sorted([
   `<${TAXON}/8RHTH> <http://rs.tdwg.org/dwc/terms/scientificName> "Saigona" .`,
   `<${TAXON}/8RHTH> <http://rs.tdwg.org/dwc/terms/taxonRank> "genus" .`,
-  `<${TAXON}/8RHTH> <http://www.w3.org/2002/07/owl#sameAs> <http://taxon-name.plazi.org/id/Animalia/Saigona> .`,
+  `<${TAXON}/8RHTH> <http://www.w3.org/2002/07/owl#sameAs> <https://taxon-name.plazi.org/id/Animalia/Saigona> .`,
   `<${TAXON}/6W7J3> <http://rs.tdwg.org/dwc/terms/scientificName> "Saigona sinensis" .`,
   `<${TAXON}/6W7J3> <http://rs.tdwg.org/dwc/terms/parentNameUsageID> <${TAXON}/8RHTH> .`,
   `<${TAXON}/GONE1> <http://rs.tdwg.org/dwc/terms/scientificName> "Obsoleta nomen" .`,
@@ -52,7 +52,7 @@ const oldSnapshot = sorted([
 const newSnapshot = sorted([
   `<${TAXON}/8RHTH> <http://rs.tdwg.org/dwc/terms/scientificName> "Saigona" .`,
   `<${TAXON}/8RHTH> <http://rs.tdwg.org/dwc/terms/taxonRank> "genus" .`,
-  `<${TAXON}/8RHTH> <http://www.w3.org/2002/07/owl#sameAs> <http://taxon-name.plazi.org/id/Animalia/Saigona> .`,
+  `<${TAXON}/8RHTH> <http://www.w3.org/2002/07/owl#sameAs> <https://taxon-name.plazi.org/id/Animalia/Saigona> .`,
   `<${TAXON}/6W7J3> <http://rs.tdwg.org/dwc/terms/scientificName> "Saigona sinensis Ôuchi, 1940" .`,
   `<${TAXON}/6W7J3> <http://rs.tdwg.org/dwc/terms/parentNameUsageID> <${TAXON}/8RHTH> .`,
   `<${TAXON}/NEW01> <http://rs.tdwg.org/dwc/terms/scientificName> "Saigona nova" .`,
@@ -212,7 +212,7 @@ Deno.test("applying the patch turns the old snapshot into the new one", async ()
   // the foreign triple in the shared graph is untouched
   assert(
     store.query(
-      `ASK { GRAPH <${GRAPH}> { <http://taxon-name.plazi.org/id/Animalia/Saigona> ?p ?o } }`,
+      `ASK { GRAPH <${GRAPH}> { <https://taxon-name.plazi.org/id/Animalia/Saigona> ?p ?o } }`,
     ),
   );
 });

src/sparql.ts

@@ -49,10 +49,10 @@ export interface SingleGraphConfig extends CommonConfig {
   /**
    * Namespace of the subjects a treatment file owns.
    *
-   * Note this is NOT `graphUriPrefix`: gg2rdf writes subjects under
-   * `http://treatment.plazi.org/id/...` while the graph names used by
-   * `graph-per-file` are `https://treatment.plazi.org/id/...`. They are
-   * different namespaces and are not interchangeable.
+   * Note this is NOT `graphUriPrefix`, even though the two spell the same
+   * since plazi/gg2rdf#34 (issue #33): this one is the namespace of the subjects gg2rdf
+   * writes, the other names the graphs `graph-per-file` loads into. They play
+   * different roles and are not interchangeable.
    */
   treatmentUriPrefix: string;
   /** Namespace of material citations, which also embed the treatment id. */
@@ -202,7 +202,7 @@ export function subjectsIn(turtle: string) {
   return [...turtle.matchAll(/^<([^>]*)>[ \t]*$/gm)].map((match) => match[1]);
 }
 
-const TAXON_NAME_NAMESPACE = "http://taxon-name.plazi.org/id/";
+const TAXON_NAME_NAMESPACE = "https://taxon-name.plazi.org/id/";
 
 /** The subjects of a gg2rdf file that can be taxon names, by namespace. Only
  * these can carry the index, so listing the others in a `VALUES` clause just
@@ -316,7 +316,7 @@ export function statementsFor(
  */
 export const SWEEPABLE_NAMESPACES = [
   TAXON_NAME_NAMESPACE,
-  "http://taxon-concept.plazi.org/id/",
+  "https://taxon-concept.plazi.org/id/",
 ];
 
 function orphanPattern(namespaces: string[]) {

src/sparql_test.ts

@@ -28,8 +28,8 @@ const config: SingleGraphConfig = {
   uploadUri: "https://example.org/sparql",
   targetGraph: TARGET,
   insertVia: "insert-data",
-  treatmentUriPrefix: "http://treatment.plazi.org/id",
-  materialCitationUriPrefix: "http://tb.plazi.org/GgServer/dwcaRecords",
+  treatmentUriPrefix: "https://treatment.plazi.org/id",
+  materialCitationUriPrefix: "https://tb.plazi.org/GgServer/dwcaRecords",
 };
 
 const A = "000040332F2853C295734E7BD4190F05";
@@ -50,25 +50,25 @@ const treatmentB = `@prefix dc: <http://purl.org/dc/elements/1.1/> .
 @prefix dwcFP: <http://filteredpush.org/ontologies/oa/dwcFP#> .
 @prefix trt: <http://plazi.org/vocab/treatment#> .
 
-<http://treatment.plazi.org/id/${B}>
+<https://treatment.plazi.org/id/${B}>
     dc:title "Saigona testensis Zheng & Chen 2021, sp. nov." ;
-    dwc:basisOfRecord <http://tb.plazi.org/GgServer/dwcaRecords/${B}.mc.1> ;
-    trt:definesTaxonConcept <http://taxon-concept.plazi.org/id/Animalia/Saigona_testensis_Zheng_2021> ;
+    dwc:basisOfRecord <https://tb.plazi.org/GgServer/dwcaRecords/${B}.mc.1> ;
+    trt:definesTaxonConcept <https://taxon-concept.plazi.org/id/Animalia/Saigona_testensis_Zheng_2021> ;
     trt:publishedIn <http://doi.org/10.3897/zookeys.1054.67004> ;
     a trt:Treatment .
 
-<http://tb.plazi.org/GgServer/dwcaRecords/${B}.mc.1>
+<https://tb.plazi.org/GgServer/dwcaRecords/${B}.mc.1>
     dwc:catalogNumber "TEST-1" ;
     a dwc:MaterialCitation .
 
-<http://taxon-concept.plazi.org/id/Animalia/Saigona_testensis_Zheng_2021>
+<https://taxon-concept.plazi.org/id/Animalia/Saigona_testensis_Zheng_2021>
     dwc:genus "Saigona" ;
-    trt:hasTaxonName <http://taxon-name.plazi.org/id/Animalia/Saigona_testensis> ;
+    trt:hasTaxonName <https://taxon-name.plazi.org/id/Animalia/Saigona_testensis> ;
     a dwcFP:TaxonConcept .
 
-<http://taxon-name.plazi.org/id/Animalia/Saigona_testensis>
+<https://taxon-name.plazi.org/id/Animalia/Saigona_testensis>
     dwc:genus "Saigona" ;
-    trt:hasParentName <http://taxon-name.plazi.org/id/Animalia/Saigona> ;
+    trt:hasParentName <https://taxon-name.plazi.org/id/Animalia/Saigona> ;
     a dwcFP:TaxonName .
 `;
 
@@ -83,7 +83,7 @@ INSERT DATA {
   GRAPH <${TARGET}> {
     <https://www.catalogueoflife.org/data/taxon/8RHTH>
       rdfs:label "Saigona" ;
-      owl:sameAs <http://taxon-name.plazi.org/id/Animalia/Saigona> .
+      owl:sameAs <https://taxon-name.plazi.org/id/Animalia/Saigona> .
   }
 }`;
 
@@ -152,7 +152,7 @@ Deno.test("graph-per-file emits the statements it always has", () => {
 Deno.test("turtleToInsertData accepts real gg2rdf output", () => {
   const store = new Store();
   store.update(turtleToInsertData(treatmentA, TARGET));
-  assert(has(store, `http://treatment.plazi.org/id/${A}`));
+  assert(has(store, `https://treatment.plazi.org/id/${A}`));
   // the file describes the treatment, the publication, one taxon concept,
   // seven taxon names and five figures
   assertEquals(subjects(store).length, 15);
@@ -174,22 +174,22 @@ Deno.test("deleting a treatment takes what it owns and nothing else", () => {
   store.update(deleteOwnedStatement(config, `data/${B}.ttl`));
 
   // gone: the treatment and the material citation it owns
-  assert(!has(store, `http://treatment.plazi.org/id/${B}`));
+  assert(!has(store, `https://treatment.plazi.org/id/${B}`));
   assert(
-    !has(store, `http://tb.plazi.org/GgServer/dwcaRecords/${B}.mc.1`),
+    !has(store, `https://tb.plazi.org/GgServer/dwcaRecords/${B}.mc.1`),
     "material citations carry the treatment id and are owned",
   );
 
   // kept: the other treatment, everything shared, and the foreign dataset
-  assert(has(store, `http://treatment.plazi.org/id/${A}`));
+  assert(has(store, `https://treatment.plazi.org/id/${A}`));
   assert(has(store, "http://doi.org/10.3897/zookeys.1054.67004"));
-  assert(has(store, "http://taxon-name.plazi.org/id/Animalia/Saigona"));
+  assert(has(store, "https://taxon-name.plazi.org/id/Animalia/Saigona"));
   assert(has(store, "https://www.catalogueoflife.org/data/taxon/8RHTH"));
 
   // and the taxon name only B referred to is left behind — that is what the
   // sweep is for, a treatment's own delete cannot tell it was the last referrer
   assert(
-    has(store, "http://taxon-name.plazi.org/id/Animalia/Saigona_testensis"),
+    has(store, "https://taxon-name.plazi.org/id/Animalia/Saigona_testensis"),
   );
 });
 
@@ -240,10 +240,10 @@ Deno.test("the sweep collects orphans, in layers, and stops at referenced ones",
   // the concepts and species names only these treatments used are gone
   for (
     const gone of [
-      "http://taxon-concept.plazi.org/id/Animalia/Saigona_baiseensis_Zheng_2021",
-      "http://taxon-concept.plazi.org/id/Animalia/Saigona_testensis_Zheng_2021",
-      "http://taxon-name.plazi.org/id/Animalia/Saigona_baiseensis",
-      "http://taxon-name.plazi.org/id/Animalia/Saigona_testensis",
+      "https://taxon-concept.plazi.org/id/Animalia/Saigona_baiseensis_Zheng_2021",
+      "https://taxon-concept.plazi.org/id/Animalia/Saigona_testensis_Zheng_2021",
+      "https://taxon-name.plazi.org/id/Animalia/Saigona_baiseensis",
+      "https://taxon-name.plazi.org/id/Animalia/Saigona_testensis",
     ]
   ) {
     assert(!has(store, gone), `${gone} should have been swept`);
@@ -253,10 +253,10 @@ Deno.test("the sweep collects orphans, in layers, and stops at referenced ones",
   // protection propagates up the parent chain
   for (
     const kept of [
-      "http://taxon-name.plazi.org/id/Animalia/Saigona",
-      "http://taxon-name.plazi.org/id/Animalia/Dictyopharidae",
-      "http://taxon-name.plazi.org/id/Animalia/Hemiptera",
-      "http://taxon-name.plazi.org/id/Animalia",
+      "https://taxon-name.plazi.org/id/Animalia/Saigona",
+      "https://taxon-name.plazi.org/id/Animalia/Dictyopharidae",
+      "https://taxon-name.plazi.org/id/Animalia/Hemiptera",
+      "https://taxon-name.plazi.org/id/Animalia",
       "https://www.catalogueoflife.org/data/taxon/8RHTH",
     ]
   ) {
@@ -298,16 +298,16 @@ function prefixesOf(store: Store, subject: string) {
 Deno.test("subjectsIn finds what a gg2rdf file describes", () => {
   const found = subjectsIn(treatmentA);
   assertEquals(found.length, 15);
-  assert(found.includes(`http://treatment.plazi.org/id/${A}`));
-  assert(found.includes("http://taxon-name.plazi.org/id/Animalia/Saigona"));
+  assert(found.includes(`https://treatment.plazi.org/id/${A}`));
+  assert(found.includes("https://taxon-name.plazi.org/id/Animalia/Saigona"));
 });
 
 Deno.test("taxonNamesIn keeps only the taxon name namespace", () => {
   const names = taxonNamesIn(treatmentA);
   assert(names.length > 0);
-  assert(names.every((s) => s.startsWith("http://taxon-name.plazi.org/id/")));
-  assert(names.includes("http://taxon-name.plazi.org/id/Animalia/Saigona"));
-  assert(!names.includes(`http://treatment.plazi.org/id/${A}`));
+  assert(names.every((s) => s.startsWith("https://taxon-name.plazi.org/id/")));
+  assert(names.includes("https://taxon-name.plazi.org/id/Animalia/Saigona"));
+  assert(!names.includes(`https://treatment.plazi.org/id/${A}`));
 });
 
 Deno.test("the index VALUES clause lists only taxon names", () => {
@@ -316,7 +316,7 @@ Deno.test("the index VALUES clause lists only taxon names", () => {
     readFile: () => treatmentA,
   });
   const values = statement.match(/VALUES \?res \{([^}]*)\}/)![1];
-  assert(values.includes("<http://taxon-name.plazi.org/id/Animalia/Saigona>"));
+  assert(values.includes("<https://taxon-name.plazi.org/id/Animalia/Saigona>"));
   assert(!values.includes("treatment.plazi.org"));
   assert(!values.includes("dwcaRecords"));
 });
@@ -353,7 +353,7 @@ Deno.test("taxomplete triples are derived for the file's taxon names", () => {
   assertEquals(
     prefixesOf(
       store,
-      "http://taxon-name.plazi.org/id/Animalia/Saigona_baiseensis",
+      "https://taxon-name.plazi.org/id/Animalia/Saigona_baiseensis",
     ),
     [
       "genusPrefix2=sa",
@@ -371,7 +371,7 @@ Deno.test("taxomplete triples are derived for the file's taxon names", () => {
 
   // ranks above genus carry no dwc:genus, so they get nothing
   assertEquals(
-    prefixesOf(store, "http://taxon-name.plazi.org/id/Animalia/Insecta"),
+    prefixesOf(store, "https://taxon-name.plazi.org/id/Animalia/Insecta"),
     [],
   );
 });
@@ -401,7 +401,7 @@ Deno.test("indexing is scoped to the file and stays idempotent", () => {
   assertEquals(
     prefixesOf(
       store,
-      "http://taxon-name.plazi.org/id/Animalia/Saigona_testensis",
+      "https://taxon-name.plazi.org/id/Animalia/Saigona_testensis",
     ),
     [],
     "B's names are not this file's to index",
@@ -452,7 +452,7 @@ Deno.test("derived triples neither protect nor survive a swept name", () => {
       })
     ) store.update(statement);
   }
-  const species = "http://taxon-name.plazi.org/id/Animalia/Saigona_baiseensis";
+  const species = "https://taxon-name.plazi.org/id/Animalia/Saigona_baiseensis";
   assert(prefixesOf(store, species).length > 0, "indexed to begin with");
 
   store.update(deleteOwnedStatement(indexing, `data/${A}.ttl`));
@@ -477,7 +477,7 @@ Deno.test("derived triples neither protect nor survive a swept name", () => {
   assertEquals(prefixesOf(store, species), [], "no dangling index entries");
   // the genus an external dataset points at keeps both its triples and its index
   assert(
-    prefixesOf(store, "http://taxon-name.plazi.org/id/Animalia/Saigona")
+    prefixesOf(store, "https://taxon-name.plazi.org/id/Animalia/Saigona")
       .length > 0,
   );
 });

src/test_deps.ts

@@ -1,6 +1,9 @@
 // Dependencies used only by tests. Kept out of deps.ts so that the docker
 // image does not have to cache them.
-export { Store } from "npm:oxigraph@0.4.11";
+// 0.5 or newer: 0.4.x evaluates the owned-subject DELETE wrongly once the
+// filter's prefixes are https:// (the SELECT with the same WHERE is fine), and
+// leaves the material citations behind.
+export { Store } from "npm:oxigraph@0.5.11";
 export {
   assert,
   assertEquals,