OPENNLP-1866: Update Dev Manual on Unicode normalization and the UAX #29 tokenizer (4-docs/7)

opennlp-docs/src/docbkx/doccat.xml

@@ -165,12 +165,34 @@ String bestCategory = categorizer.getBestCategory(outcomes);]]>
 File vocab = new File("/path/to/vocab.txt");
 Map<Integer, String> categories = new HashMap<>();
 String[] inputText = new String[]{"My input text is great."};
-final DocumentCategorizerDL myCategorizer = new DocumentCategorizerDL(model, vocab, categories);
+final DocumentCategorizerDL myCategorizer = new DocumentCategorizerDL(
+    model, vocab, categories, new AverageClassificationScoringStrategy(), new InferenceOptions());
 double[] outcomes = myCategorizer.categorize(inputText);
 String category = myCategorizer.getBestCategory(outcomes);]]>
 				</programlisting>
 				For additional examples, refer to the <code>DocumentCategorizerDLEval</code> class.
 			</para>
+			<para>
+				Like <code>NameFinderDL</code>, long input is split into overlapping chunks on the full
+				Unicode <code>White_Space</code> set rather than Java's <code>\s</code>, so text copied
+				from PDFs, the web, or multilingual sources tokenizes consistently. Optional
+				preprocessing through <code>InferenceOptions</code> is off by default:
+				<code>setNormalizeWhitespace(true)</code> maps each Unicode whitespace code point to an
+				ASCII space, and <code>setNormalizeDashes(true)</code> maps Unicode dashes to the ASCII
+				hyphen-minus. Whitespace folding is a one-to-one replacement that preserves character
+				offsets, and so is dash folding for Basic Multilingual Plane dashes; a
+				supplementary-plane dash shrinks from two UTF-16 units to one and shifts later offsets
+				(see <xref linkend="tools.normalizer.dl"/>). See
+				<xref linkend="tools.normalizer"/> for the shared <code>CharClass</code> engine and the
+				full normalization library.
+			</para>
+			<programlisting language="java">
+<![CDATA[InferenceOptions options = new InferenceOptions();
+options.setNormalizeWhitespace(true);
+options.setNormalizeDashes(true);
+DocumentCategorizerDL categorizer = new DocumentCategorizerDL(
+    model, vocab, categories, new AverageClassificationScoringStrategy(), options);]]>
+			</programlisting>
 		</section>
 	</section>
 

opennlp-docs/src/docbkx/introduction.xml

@@ -303,7 +303,8 @@ Arguments description:
                 and <xref linkend="tools.doccat">Document Categorizer</xref>. This allows models trained by other frameworks
                 such as PyTorch and Tensorflow to be used by OpenNLP. The documentation for
                 each of the OpenNLP components that supports ONNX models describes how to
-                use ONNX models for inference.
+                use ONNX models for inference. DL inference uses Unicode-aware text chunking and
+                optional input normalization; see <xref linkend="tools.normalizer.dl"/>.
             </para>
             <note>
                 <para>

opennlp-docs/src/docbkx/namefinder.xml

@@ -155,13 +155,70 @@ Span[] nameSpans = nameFinder.find(sentence);]]>
 					<programlisting language="java">
 <![CDATA[File model = new File("/path/to/model.onnx");
 File vocab = new File("/path/to/vocab.txt");
-Map<Integer, String> categories = new HashMap<>();
-String[] tokens = new String[]{"George", "Washington", "was", "president", "of", "the", "United", "States", "."};
-NameFinderDL nameFinderDL = new NameFinderDL(model, vocab, false, getIds2Labels());
-Span[] spans = nameFinderDL.find(tokens);]]>
+// Maps every model output index to its BIO label. This must be exhaustive over the model's
+// output indices; an unmapped predicted index raises IllegalStateException during find().
+Map<Integer, String> ids2Labels = new HashMap<>();
+ids2Labels.put(0, "O");
+ids2Labels.put(1, "B-PER");
+ids2Labels.put(2, "I-PER");
+ids2Labels.put(3, "B-ORG");
+ids2Labels.put(4, "I-ORG");
+ids2Labels.put(5, "B-LOC");
+ids2Labels.put(6, "I-LOC");
+ids2Labels.put(7, "B-MISC");
+ids2Labels.put(8, "I-MISC");
+SentenceDetector sentenceDetector =
+    new SentenceDetectorME(new SentenceModel(
+        new File("/path/to/opennlp-en-ud-ewt-sentence-1.3-2.5.4.bin")));
+String[] tokens = {"George", "Washington", "was", "president", "of", "the", "United", "States", "."};
+NameFinderDL nameFinderDL = new NameFinderDL(model, vocab, ids2Labels, sentenceDetector);
+// findInOriginal returns spans in the original input's coordinates.
+Span[] spans = nameFinderDL.findInOriginal(tokens);]]>
 					</programlisting>
 					For additional examples, refer to the <code>NameFinderDLEval</code> class.
 				</para>
+				<para>
+					Long input text is split into overlapping chunks on the full Unicode
+					<code>White_Space</code> set before WordPiece tokenization, so spacing such as a
+					no-break space or the CJK ideographic space is recognized as a delimiter. After
+					inference, reconstructed entity text is matched back to the caller's original input
+					with a Unicode-aware cursor scan (not a regular expression), so
+					<code>Span#getCoveredText(...)</code> returns the source text even when WordPiece
+					rejoins sub-tokens with spaces or when the source uses non-ASCII whitespace between
+					tokens.
+				</para>
+				<para>
+					<code>findInOriginal</code> is declared by the <code>OffsetMappingNameFinder</code>
+					capability interface that <code>NameFinderDL</code> implements, so a caller holding a
+					plain <code>TokenNameFinder</code> can detect the offset-mapping capability with a
+					<code>finder instanceof OffsetMappingNameFinder</code> check (no reflection) and fall
+					back to token-index spans otherwise.
+				</para>
+				<para>
+					Optional preprocessing of the joined input text is available through
+					<code>InferenceOptions</code> and is off by default:
+					<code>setNormalizeWhitespace(true)</code> folds each Unicode whitespace character to
+					an ASCII space, and <code>setNormalizeDashes(true)</code> folds Unicode dashes to the
+					ASCII hyphen-minus. Whitespace folding is one code point to one character and
+					preserves offsets, and so is dash folding for Basic Multilingual Plane dashes; a
+					supplementary-plane dash shrinks from two UTF-16 units to one and shifts later
+					offsets, which <code>findInOriginal</code> maps back across (see
+					<xref linkend="tools.normalizer.dl"/>). Full details, the underlying
+					<code>CharClass</code> engine, and the broader normalization pipeline are documented
+					in <xref linkend="tools.normalizer"/>.
+				</para>
+				<para>
+					The following listing reuses <code>model</code>, <code>vocab</code>,
+					<code>ids2Labels</code>, <code>sentenceDetector</code>, and <code>tokens</code> from
+					the example above and only shows the <code>InferenceOptions</code> wiring:
+				</para>
+				<programlisting language="java">
+<![CDATA[InferenceOptions options = new InferenceOptions();
+options.setNormalizeWhitespace(true);
+options.setNormalizeDashes(true);
+NameFinderDL finder = new NameFinderDL(model, vocab, ids2Labels, options, sentenceDetector);
+Span[] spans = finder.findInOriginal(tokens);]]>
+				</programlisting>
 			</section>
 	</section>
 	</section>