diff --git a/pom.xml b/pom.xml index 7eceab29..69c8e87a 100644 --- a/pom.xml +++ b/pom.xml @@ -1,220 +1,226 @@ - 4.0.0 - BURP - BURP - 0.1.2-SNAPSHOT - Basic and Unassuming RML Processor + xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" + xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/xsd/maven-4.0.0.xsd"> + 4.0.0 + BURP + BURP + 0.1.2-SNAPSHOT + Basic and Unassuming RML Processor - - 17 - 17 - UTF-8 - + + 17 + 17 + UTF-8 + - - - jitpack.io - https://jitpack.io - - + + + jitpack.io + https://jitpack.io + + - - src/main/java - - - src/main/resources - - **/*.ttl - - - + + src/main/java + + + src/main/resources + + **/*.ttl + + + - - - maven-compiler-plugin - 3.8.1 - - 17 - - + + + maven-compiler-plugin + 3.8.1 + + 17 + + - - org.apache.maven.plugins - maven-surefire-plugin - 3.2.5 - + + org.apache.maven.plugins + maven-surefire-plugin + 3.2.5 + - - org.apache.maven.plugins - maven-jar-plugin - 3.3.0 - - - - true - dependency/ - burp.Main - - - - + + org.apache.maven.plugins + maven-jar-plugin + 3.3.0 + + + + true + dependency/ + burp.Main + + + + - - org.apache.maven.plugins - maven-shade-plugin - 2.4.3 - - - package - - shade - - - true - - - burp.Main - - - - - false - - - - - *:* - - META-INF/*.SF - META-INF/*.DSA - META-INF/*.RSA - - - - burp - - - - - - + + org.apache.maven.plugins + maven-shade-plugin + 2.4.3 + + + package + + shade + + + true + + + burp.Main + + + + + false + + + + + *:* + + META-INF/*.SF + META-INF/*.DSA + META-INF/*.RSA + + + + burp + + + + + + - - - info.picocli - picocli - 4.7.5 - - - org.apache.jena - jena-arq - 4.10.0 - - - com.opencsv - opencsv - 5.9 - - - com.jayway.jsonpath - json-path - 2.9.0 - - - net.minidev - json-smart - 2.5.0 - - - org.apache.jena - jena-shacl - 4.10.0 - - - mysql - mysql-connector-java - 8.0.33 - - - org.postgresql - postgresql - 42.7.2 - - - com.microsoft.sqlserver - mssql-jdbc - 12.6.1.jre11 - - - org.xerial - sqlite-jdbc - 3.46.0.0 - - - org.apache.commons - commons-text - 1.11.0 - + + + info.picocli + picocli + 4.7.5 + + + org.apache.jena + jena-arq + 4.10.0 + + + com.opencsv + opencsv + 5.9 + + + com.jayway.jsonpath + json-path + 2.9.0 + + + net.minidev + json-smart + 2.5.0 + + + org.apache.jena + jena-shacl + 4.10.0 + + + mysql + mysql-connector-java + 8.0.33 + + + org.postgresql + postgresql + 42.7.2 + + + com.microsoft.sqlserver + mssql-jdbc + 12.6.1.jre11 + + + org.xerial + sqlite-jdbc + 3.46.0.0 + + + org.apache.commons + commons-text + 1.11.0 + - - - org.junit.jupiter - junit-jupiter-engine - 5.10.2 - test - - - org.junit.jupiter - junit-jupiter-api - 5.10.2 - test - - - org.testcontainers - junit-jupiter - 1.19.5 - test - - - org.testcontainers - postgresql - 1.19.5 - test - - - org.testcontainers - mysql - 1.19.5 - test - - - org.apache.jena - jena-fuseki-main - 4.10.0 - test - + + + org.junit.jupiter + junit-jupiter-engine + 5.10.2 + test + + + org.junit.jupiter + junit-jupiter-api + 5.10.2 + test + + + org.testcontainers + junit-jupiter + 1.19.5 + test + + + org.junit.jupiter + junit-jupiter-params + 5.10.2 + test + + + org.testcontainers + postgresql + 1.19.5 + test + + + org.testcontainers + mysql + 1.19.5 + test + + + org.apache.jena + jena-fuseki-main + 4.10.0 + test + - - org.tukaani - xz - 1.9 - - - com.univocity - univocity-parsers - 2.9.1 - - - com.github.candil-data-fabric - JNC - xpath-namespace-SNAPSHOT - - + + org.tukaani + xz + 1.9 + + + com.univocity + univocity-parsers + 2.9.1 + + + com.github.candil-data-fabric + JNC + xpath-namespace-SNAPSHOT + + diff --git a/src/main/java/burp/Generator.java b/src/main/java/burp/Generator.java new file mode 100644 index 00000000..d1900afa --- /dev/null +++ b/src/main/java/burp/Generator.java @@ -0,0 +1,205 @@ +package burp; + +import burp.model.GraphMap; +import burp.model.Iteration; +import burp.model.ObjectMap; +import burp.model.PredicateMap; +import burp.model.PredicateObjectMap; +import burp.model.ReferencingObjectMap; +import burp.model.SubjectMap; +import burp.model.TriplesMap; +import burp.model.gathermaputil.SubGraph; +import burp.vocabularies.RML; +import java.util.ArrayList; +import java.util.Iterator; +import java.util.List; +import java.util.function.Consumer; +import org.apache.jena.graph.Node; +import org.apache.jena.rdf.model.RDFNode; +import org.apache.jena.rdf.model.Resource; +import org.apache.jena.sparql.core.Quad; +import org.apache.jena.vocabulary.RDF; + +/** + * Generates quadruples by executing supplied RML triples maps. + */ +public class Generator { + + private final List def = List.of(RML.defaultGraph); + + /** + * Executes the supplied RML triples maps and streams the generated RDF quads to the provided consumer. + *

+ * No dataset is materialized in memory. Each generated quad is emitted immediately by invoking {@link Consumer#accept(Object)} on the supplied consumer. + * The order of emitted quads follows the order in which triples maps, logical source iterations, subject maps, and predicate-object maps are processed by + * the generator. A {@code Consumer} is intended for side-effecting operations such as writing, indexing, or collecting generated data. + * + * @param triplesmaps the triples maps to execute + * @param baseIRI base IRI used to resolve relative IRIs produced by the mapping + * @param defaultGraph the default graph to be used + * @param consumer callback invoked once for every generated quad + */ + public void generate(List triplesmaps, + String baseIRI, + Node defaultGraph, + Consumer consumer) { + + for (TriplesMap tm : triplesmaps) { + + SubjectMap sm = tm.subjectMap; + List sgm = sm.graphMaps; + + Iterator iter = tm.logicalSource.iterator(); + while (iter.hasNext()) { + + Iteration i = iter.next(); + + // subject graphs + List sgs = sgm.isEmpty() + ? def + : new ArrayList<>(); + + for (GraphMap gm : sgm) { + sgs.addAll(gm.generateTerms(i, baseIRI)); + } + + // subjects + List subjects = new ArrayList<>(); + + if (!sm.isGatherMap()) { + subjects.addAll(sm.generateTerms(i, baseIRI)); + } else { + for (SubGraph subgraph : sm.generateGatherMapGraphs(i, baseIRI)) { + subjects.add(subgraph.node); + emitSubGraph(subgraph, sgs, defaultGraph, consumer); + } + } + + // rdf:type from subject map + emitSubjectTypes(sm.classes, subjects, sgs, defaultGraph, consumer); + + // predicate-object maps + for (PredicateObjectMap pom : tm.predicateObjectMaps) { + + List pogs = new ArrayList<>(); + for (GraphMap gm : pom.graphMaps) { + pogs.addAll(gm.generateTerms(i, baseIRI)); + } + + List graphs = def; + + if (!sgm.isEmpty() || !pogs.isEmpty()) { + pogs.addAll(!sgm.isEmpty() ? sgs : new ArrayList<>()); + graphs = pogs; + } + + List predicates = new ArrayList<>(); + for (PredicateMap pm : pom.predicateMaps) { + predicates.addAll(pm.generateTerms(i, baseIRI)); + } + + List objects = new ArrayList<>(); + + for (ObjectMap om : pom.objectMaps) { + if (!om.isGatherMap()) { + objects.addAll(om.generateTerms(i, baseIRI)); + } else { + for (SubGraph subgraph : om.generateGatherMapGraphs(i, baseIRI)) { + objects.add(subgraph.node); + emitSubGraph(subgraph, graphs, defaultGraph, consumer); + } + } + } + + for (ReferencingObjectMap rom : pom.refObjectMaps) { + objects.addAll(rom.generateTerms(i, baseIRI)); + } + + emitTriples(subjects, predicates, objects, graphs, defaultGraph, consumer); + } + } + } + } + + /* ---------------- Quad emission helpers ---------------- */ + private void emitTriples(List subjects, + List predicates, + List objects, + List graphs, + Node defaultGraph, + Consumer consumer) { + + for (RDFNode s : subjects) { + Node sn = s.asNode(); + + for (RDFNode p : predicates) { + Node pn = p.asNode(); + + for (RDFNode o : objects) { + Node on = o.asNode(); + + for (RDFNode g : graphs) { + Node gn = toGraphNode(g, defaultGraph); + consumer.accept(Quad.create(gn, sn, pn, on)); + } + } + } + } + } + + private void emitSubjectTypes(List classes, + List subjects, + List graphs, + Node defaultGraph, + Consumer consumer) { + + Node rdfType = RDF.type.asNode(); + + for (RDFNode s : subjects) { + Node sn = s.asNode(); + + for (Resource c : classes) { + Node cn = c.asNode(); + + for (RDFNode g : graphs) { + Node gn = toGraphNode(g, defaultGraph); + consumer.accept(Quad.create(gn, sn, rdfType, cn)); + } + } + } + } + + /* ---------------- SubGraph handling ---------------- */ + private void emitSubGraph(SubGraph subgraph, + List graphs, + Node defaultGraph, + Consumer consumer) { + + Node gn = null; + + for (RDFNode g : graphs) { + gn = toGraphNode(g, defaultGraph); + + Node finalGn = gn; + + // emit all triples from subgraph.model into quads + subgraph.model.listStatements().forEachRemaining(st -> { + consumer.accept(Quad.create( + finalGn, + st.getSubject().asNode(), + st.getPredicate().asNode(), + st.getObject().asNode() + )); + }); + } + } + + /* ---------------- graph conversion ---------------- */ + private Node toGraphNode(RDFNode g, Node defaultGraph) { + if (g.equals(RML.defaultGraph)) { + return defaultGraph; + } + return g.asNode(); + } + +} diff --git a/src/test/java/burp/TestGenerator.java b/src/test/java/burp/TestGenerator.java new file mode 100644 index 00000000..2eed972b --- /dev/null +++ b/src/test/java/burp/TestGenerator.java @@ -0,0 +1,166 @@ +package burp; + +import burp.model.TriplesMap; +import burp.parse.Parse; +import java.io.File; +import java.util.List; +import org.apache.jena.rdf.model.Model; +import org.apache.jena.riot.Lang; +import org.apache.jena.riot.RDFDataMgr; +import org.apache.jena.sparql.core.DatasetGraph; +import org.apache.jena.sparql.core.DatasetGraphFactory; +import org.apache.jena.sparql.core.Quad; +import org.junit.jupiter.api.Assertions; +import static org.junit.jupiter.api.Assertions.assertTrue; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; + +/** + * + */ +public class TestGenerator { + + private static String base = "./src/test/resources/rml-core/"; + + @ParameterizedTest + @ValueSource(strings = { + "RMLTC0000-CSV", + "RMLTC0000-JSON", + "RMLTC0000-XML", + "RMLTC0001a-CSV", + "RMLTC0001a-JSON", + "RMLTC0001a-XML", + "RMLTC0001b-CSV", + "RMLTC0001b-JSON", + "RMLTC0001b-XML", + "RMLTC0002a-CSV", + "RMLTC0002a-JSON", + "RMLTC0002a-XML", + "RMLTC0002b-CSV", + "RMLTC0002b-JSON", + "RMLTC0002b-XML", + "RMLTC0003c-CSV", + "RMLTC0003c-JSON", + "RMLTC0003c-XML", + "RMLTC0004a-CSV", + "RMLTC0004a-JSON", + "RMLTC0004a-XML", + "RMLTC0005a-CSV", + "RMLTC0005a-JSON", + "RMLTC0005a-XML", + "RMLTC0006a-CSV", + "RMLTC0006a-JSON", + "RMLTC0006a-XML", + "RMLTC0007a-CSV", + "RMLTC0007a-JSON", + "RMLTC0007a-XML", + "RMLTC0007b-CSV", + "RMLTC0007b-JSON", + "RMLTC0007b-XML", + "RMLTC0007c-CSV", + "RMLTC0007c-JSON", + "RMLTC0007c-XML", + "RMLTC0007d-CSV", + "RMLTC0007d-JSON", + "RMLTC0007d-XML", + "RMLTC0007e-CSV", + "RMLTC0007e-JSON", + "RMLTC0007e-XML", + "RMLTC0007f-CSV", + "RMLTC0007f-JSON", + "RMLTC0007f-XML", + "RMLTC0007g-CSV", + "RMLTC0007g-JSON", + "RMLTC0007g-XML", + "RMLTC0008a-CSV", + "RMLTC0008a-JSON", + "RMLTC0008a-XML", + "RMLTC0008b-CSV", + "RMLTC0008b-JSON", + "RMLTC0008b-XML", + "RMLTC0008c-CSV", + "RMLTC0008c-JSON", + "RMLTC0008c-XML", + "RMLTC0009a-CSV", + "RMLTC0009a-JSON", + "RMLTC0009a-XML", + "RMLTC0009b-CSV", + "RMLTC0009b-JSON", + "RMLTC0009b-XML", + "RMLTC0010a-CSV", + "RMLTC0010a-JSON", + "RMLTC0010b-CSV", + "RMLTC0010b-JSON", + "RMLTC0010b-XML", + "RMLTC0010c-CSV", + "RMLTC0010c-JSON", + "RMLTC0010c-XML", + "RMLTC0011b-CSV", + "RMLTC0011b-JSON", + "RMLTC0011b-XML", + "RMLTC0012a-CSV", + "RMLTC0012a-JSON", + "RMLTC0012a-XML", + "RMLTC0012b-CSV", + "RMLTC0012b-JSON", + "RMLTC0012b-XML", + "RMLTC0013a-JSON", + "RMLTC0015a-CSV", + "RMLTC0015a-JSON", + "RMLTC0015a-XML", + "RMLTC0019a-CSV", + "RMLTC0019a-JSON", + "RMLTC0019a-XML", + "RMLTC0020a-CSV", + "RMLTC0020a-JSON", + "RMLTC0020a-XML", + "RMLTC0021a-CSV", + "RMLTC0021a-JSON", + "RMLTC0021a-XML", + //"RMLTC0022a-CSV", + //"RMLTC0022b-CSV", + //"RMLTC0022c-CSV", + "RMLTC0023a-XML", + //"RMLTC0026a-JSON", + "RMLTC0027-XML" + }) + public void rmlTest(String folder) throws Exception { + String m = new File(base + folder, "mapping.ttl").getAbsolutePath(); + + // Parse the mapping file + List triplesmaps = Parse.parseMappingFile(m); + + DatasetGraph actual = DatasetGraphFactory.create(); + + String baseIRI = "http://example.com/base/"; + + Generator generator = new Generator(); + + try { + generator.generate(triplesmaps, baseIRI, Quad.defaultGraphIRI, quad -> { + actual.add(quad); + }); + } catch(Exception e) { + System.out.println(folder); + Assertions.fail(e); + } + + String o = new File(base + folder, "output.nq").getAbsolutePath(); + Model expected = RDFDataMgr.loadModel(o); + + boolean b = actual.getDefaultGraph().isIsomorphicWith(expected.getGraph()); + + if(!b) { + System.out.println(folder); + System.out.println("Expected:"); + expected.write(System.out, "Turtle"); + System.out.println("---"); + System.out.println("Actual:"); + RDFDataMgr.write(System.out, actual, Lang.TRIG); + System.out.println("==============="); + } + + assertTrue(b); + } + +}