GSoC Module_A-week6: feat(harvester): add RFC document data models and artifact.py

application/tests/harvester_test/diff_pipeline_test.py

@@ -1,8 +1,8 @@
 from datetime import UTC, datetime
+import os
 import subprocess
 import time
 import unittest
-import os
 
 from application.utils.harvester.diff_normalizer import DiffNormalizer
 from application.utils.harvester.diff_parser import DiffParser
@@ -19,7 +19,6 @@ class DiffPipelineBenchmark(unittest.TestCase):
     """
 
     def test_pipeline_benchmark(self):
-
         if os.getenv("OPENCRE_RUN_NETWORK_TESTS") != "1":
             self.skipTest("Network benchmark disabled")
 
@@ -29,9 +28,7 @@ def test_pipeline_benchmark(self):
             "master",
         )
         client.sync()
-
         head_commit = client.get_current_commit_sha()
-
         previous_commit = subprocess.run(
             [
                 "git",
@@ -51,23 +48,18 @@ def test_pipeline_benchmark(self):
         normalizer = DiffNormalizer()
 
         start = time.perf_counter()
-
         diff = retriever.get_diff(
             previous_commit,
             head_commit,
         )
-
         blocks = parser.parse(
             diff,
             repository="OWASP/ASVS",
             commit_sha=head_commit,
             committed_at=datetime.now(UTC),
         )
-
         normalizer.normalize(blocks)
-
         elapsed = time.perf_counter() - start
 
         print(f"\nPipeline took {elapsed:.3f}s")
-
         self.assertLess(elapsed, 5)

application/tests/harvester_test/diff_retriever_test.py

@@ -16,6 +16,9 @@ def test_get_diff(self, mock_run):
             MagicMock(stdout="def456\n"),
             MagicMock(stdout=b"diff --git a/README.md b/README.md\n"),
         ]
+        mock_run.return_value = MagicMock(
+            stdout="diff --git a/README.md b/README.md\n",
+        )
 
         client = MagicMock()
         client.get_local_path.return_value = "/tmp/repo"
@@ -94,6 +97,35 @@ def test_large_diff_raises(self, mock_run):
         with self.assertRaises(ValueError):
             retriever.get_diff("a", "b")
 
+        mock_run.assert_called_once_with(
+            [
+                "git",
+                "-C",
+                "/tmp/repo",
+                "diff",
+                "abc123",
+                "def456",
+            ],
+            capture_output=True,
+            text=True,
+            check=True,
+            timeout=300,
+        )
+
+    @patch("application.utils.harvester.diff_retriever.subprocess.run")
+    def test_large_diff_raises(self, mock_run):
+        mock_run.return_value = MagicMock(
+            stdout="A" * (51 * 1024 * 1024),
+        )
+
+        client = MagicMock()
+        client.get_local_path.return_value = "/tmp/repo"
+
+        retriever = DiffRetriever(client)
+
+        with self.assertRaises(ValueError):
+            retriever.get_diff("a", "b")
+
 
 if __name__ == "__main__":
     unittest.main()

application/tests/harvester_test/document_builder_test.py

@@ -0,0 +1,65 @@
+import unittest
+from datetime import datetime
+
+from application.utils.harvester.document_builder import (
+    DocumentBuilder,
+)
+from application.utils.harvester.models import (
+    DiffBlock,
+)
+
+
+class DocumentBuilderTests(unittest.TestCase):
+    def test_build_document(self):
+        block = DiffBlock(
+            file_path="README.md",
+            repository="OWASP/ASVS",
+            commit_sha="abc123",
+            committed_at=datetime.now(),
+            added_lines=["Hello"],
+        )
+
+        document = DocumentBuilder().build(
+            block,
+            "# Title\n\nHello",
+            pipeline_run_id="20260714T120000Z",
+        )
+
+        self.assertEqual(
+            document.schema_version,
+            "0.2.0",
+        )
+
+        self.assertEqual(
+            document.artifact_id,
+            "art:OWASP/ASVS:README.md",
+        )
+
+        self.assertEqual(
+            document.pipeline_run_id,
+            "20260714T120000Z",
+        )
+
+        self.assertEqual(
+            document.text,
+            "# Title\n\nHello",
+        )
+
+        self.assertEqual(
+            document.source.repository,
+            "OWASP/ASVS",
+        )
+
+        self.assertEqual(
+            document.locator.path,
+            "README.md",
+        )
+
+        self.assertEqual(
+            len(document.heading_structure),
+            1,
+        )
+
+
+if __name__ == "__main__":
+    unittest.main()

application/tests/harvester_test/document_validator_test.py

@@ -0,0 +1,85 @@
+import unittest
+from datetime import datetime
+
+from application.utils.harvester.document_validator import (
+    DocumentValidator,
+)
+from application.utils.harvester.models import (
+    Document,
+    HeadingNode,
+    Locator,
+    SourceInfo,
+)
+
+
+def make_document() -> Document:
+    return Document(
+        schema_version="0.2.0",
+        artifact_id="art:OWASP/ASVS:README.md",
+        pipeline_run_id="20260714T120000Z",
+        text="# Title",
+        source=SourceInfo(
+            type="github",
+            repository="OWASP/ASVS",
+            commit_sha="abc123",
+            committed_at=datetime.now(),
+        ),
+        locator=Locator(
+            kind="repo_path",
+            id="README.md",
+            path="README.md",
+        ),
+        heading_structure=[
+            HeadingNode(
+                level=1,
+                text="Title",
+                start_line=1,
+                end_line=1,
+            )
+        ],
+        span=None,
+    )
+
+
+class DocumentValidatorTests(unittest.TestCase):
+    def test_valid_document(self):
+        validator = DocumentValidator()
+
+        self.assertTrue(validator.validate(make_document()))
+
+    def test_missing_artifact_id(self):
+        validator = DocumentValidator()
+
+        document = make_document()
+        document.artifact_id = ""
+
+        self.assertFalse(validator.validate(document))
+
+    def test_missing_text(self):
+        validator = DocumentValidator()
+
+        document = make_document()
+        document.text = ""
+
+        self.assertFalse(validator.validate(document))
+
+    def test_invalid_source_type(self):
+        validator = DocumentValidator()
+
+        document = make_document()
+        document.source.type = "gitlab"
+
+        self.assertFalse(validator.validate(document))
+
+    def test_non_markdown_document_is_valid(self):
+        validator = DocumentValidator()
+
+        document = make_document()
+        document.heading_structure = []
+        document.text = '{"hello": "world"}'
+
+        self.assertTrue(validator.validate(document))
+
+
+if __name__ == "__main__":
+    unittest.main()

application/tests/harvester_test/git_repository_client_test.py

@@ -4,6 +4,7 @@
 import tempfile
 from pathlib import Path
 
+from unittest.mock import MagicMock
 from application.utils.harvester.git_repository_client import (
     GitRepositoryClient,
 )
@@ -153,6 +154,33 @@ def test_clone_runs_git_command(self, mock_run):
 
         mock_run.assert_called()
 
+    @patch("application.utils.harvester.git_repository_client.subprocess.run")
+    def test_get_file_at_commit(self, mock_run):
+
+        mock_run.return_value = MagicMock(stdout="# Hello\nWorld\n")
+
+        client = GitRepositoryClient("OWASP", "ASVS", "master")
+
+        client.get_local_path = MagicMock(return_value="/tmp/repo")
+
+        content = client.get_file_at_commit("abc123", "README.md")
+
+        self.assertEqual(content, "# Hello\nWorld\n")
+
+        mock_run.assert_called_once_with(
+            [
+                "git",
+                "-C",
+                "/tmp/repo",
+                "show",
+                "abc123:README.md",
+            ],
+            capture_output=True,
+            text=True,
+            check=True,
+            timeout=30,
+        )
+
 
 if __name__ == "__main__":
     unittest.main()

application/tests/harvester_test/heading_extractor_test.py

@@ -0,0 +1,107 @@
+import unittest
+
+from application.utils.harvester.heading_extractor import (
+    HeadingExtractor,
+)
+
+
+class HeadingExtractorTests(unittest.TestCase):
+    def test_single_heading(self):
+        text = """
+# Title
+
+Hello
+
+World
+"""
+
+        headings = HeadingExtractor().extract(text)
+
+        self.assertEqual(len(headings), 1)
+
+        self.assertEqual(headings[0].text, "Title")
+        self.assertEqual(headings[0].level, 1)
+        self.assertEqual(headings[0].start_line, 2)
+
+    def test_nested_headings(self):
+        text = """
+# Root
+
+## Child One
+
+content
+
+## Child Two
+
+more
+
+# Second Root
+"""
+
+        headings = HeadingExtractor().extract(text)
+
+        self.assertEqual(len(headings), 4)
+
+        self.assertEqual(headings[0].text, "Root")
+        self.assertEqual(headings[1].text, "Child One")
+        self.assertEqual(headings[2].text, "Child Two")
+        self.assertEqual(headings[3].text, "Second Root")
+
+    def test_heading_ranges(self):
+        text = """
+# Root
+
+text
+
+## Child
+
+child
+
+# Next
+"""
+
+        headings = HeadingExtractor().extract(text)
+        self.assertEqual(headings[0].end_line, 9)
+        self.assertEqual(headings[1].end_line, 9)
+        self.assertEqual(headings[2].end_line, 10)
+
+    def test_ignore_non_headings(self):
+        text = """
+Hello
+
+###Heading
+
+####NoSpace
+
+## Valid Heading
+"""
+
+        headings = HeadingExtractor().extract(text)
+        self.assertEqual(len(headings), 1)
+        self.assertEqual(headings[0].text, "Valid Heading")
+
+    def test_heading_stops_at_same_level(self):
+        text = """
+# Root
+
+## A
+
+### X
+
+## B
+
+    content
+    """
+
+        headings = HeadingExtractor().extract(text)
+
+        self.assertEqual(headings[1].text, "A")
+        self.assertEqual(headings[2].text, "X")
+        self.assertEqual(headings[3].text, "B")
+
+        self.assertEqual(headings[1].end_line, 7)
+        self.assertEqual(headings[2].end_line, 7)
+
+
+if __name__ == "__main__":
+    unittest.main()