public interface Tokenizer extends Function<String, String[]> {
    String[] split(String text);   // tokenize text into words/tokens
}

public interface SentenceSplitter extends Function<String, String[]> {
    String[] split(String text);   // segment text into sentences
}

public interface ParagraphSplitter extends Function<String, String[]> {
    String[] split(String text);   // segment text into paragraphs
}

SentenceSplitter splitter = SimpleSentenceSplitter.getInstance();
Tokenizer tokenizer       = new SimpleTokenizer();

// Use as Function in a stream
String[] sentences = splitter.apply(text);
String[][] tokens  = Arrays.stream(sentences)
        .map(tokenizer)
        .toArray(String[][]::new);

import smile.nlp.tokenizer.SimpleTokenizer;

SimpleTokenizer tokenizer = new SimpleTokenizer();

String[] tokens = tokenizer.split(
    "Dr. Smith won't attend the conference, but she'll send her notes.");

System.out.println(java.util.Arrays.toString(tokens));
// [Dr., Smith, will, not, attend, the, conference, ,, but, she, 'll, send, her, notes, .]

SimpleTokenizer tokenizer = new SimpleTokenizer();

// Commas inside numbers are not split
System.out.println(Arrays.toString(tokenizer.split("Population is 2,500,000.")));
// [Population, is, 2,500,000, .]

// Ellipsis is separated
System.out.println(Arrays.toString(tokenizer.split("Wait... then go.")));
// [Wait, ..., then, go, .]

import smile.nlp.tokenizer.PennTreebankTokenizer;

PennTreebankTokenizer tokenizer = PennTreebankTokenizer.getInstance();

String[] tokens = tokenizer.split("They couldn't have known.");
System.out.println(java.util.Arrays.toString(tokens));
// [They, could, n't, have, known, .]

import smile.nlp.tokenizer.BreakIteratorTokenizer;
import java.util.Locale;

// Default locale
BreakIteratorTokenizer tokenizer = new BreakIteratorTokenizer();
System.out.println(java.util.Arrays.toString(tokenizer.split("Hello, world!")));

// Explicit locale
BreakIteratorTokenizer frTokenizer = new BreakIteratorTokenizer(Locale.FRENCH);
System.out.println(java.util.Arrays.toString(frTokenizer.split("Bonjour, le monde!")));

ThreadLocal<BreakIteratorTokenizer> tlTokenizer =
        ThreadLocal.withInitial(BreakIteratorTokenizer::new);

// In each thread:
BreakIteratorTokenizer tokenizer = tlTokenizer.get();
String[] tokens = tokenizer.split(text);

import smile.nlp.tokenizer.SimpleSentenceSplitter;

SimpleSentenceSplitter splitter = SimpleSentenceSplitter.getInstance();

String paragraph =
    "Dr. Smith attended the conf. in Jan. He presented his findings. "
  + "Was the result surprising? Absolutely!";

for (String sentence : splitter.split(paragraph)) {
    System.out.println(sentence);
}
// Dr. Smith attended the conf. in Jan.
// He presented his findings.
// Was the result surprising?
// Absolutely!

import smile.nlp.tokenizer.BreakIteratorSentenceSplitter;
import java.util.Locale;

// Default locale
BreakIteratorSentenceSplitter splitter = new BreakIteratorSentenceSplitter();

// Specific locale
BreakIteratorSentenceSplitter deSplitter =
        new BreakIteratorSentenceSplitter(Locale.GERMAN);

for (String sentence : deSplitter.split("Das ist ein Test. Und noch ein Satz.")) {
    System.out.println(sentence);
}
// Das ist ein Test.
// Und noch ein Satz.

import smile.nlp.tokenizer.SimpleParagraphSplitter;

SimpleParagraphSplitter splitter = SimpleParagraphSplitter.getInstance();

String document =
    "First paragraph with multiple sentences. It continues here.\n\n"
  + "Second paragraph begins after the blank line.\n\n"
  + "Third paragraph.";

for (String para : splitter.split(document)) {
    System.out.println("PARAGRAPH: " + para);
}
// PARAGRAPH: First paragraph with multiple sentences. It continues here.
// PARAGRAPH: Second paragraph begins after the blank line.
// PARAGRAPH: Third paragraph.

import smile.nlp.tokenizer.*;
import smile.nlp.stemmer.PorterStemmer;
import smile.nlp.pos.*;

// ── Splitters & tokenizer ────────────────────────────────────────────
ParagraphSplitter paragraphSplitter = SimpleParagraphSplitter.getInstance();
SentenceSplitter  sentenceSplitter  = SimpleSentenceSplitter.getInstance();
Tokenizer         tokenizer         = new SimpleTokenizer();
HMMPOSTagger      tagger            = HMMPOSTagger.getDefault();

ThreadLocal<PorterStemmer> tlStemmer = ThreadLocal.withInitial(PorterStemmer::new);

// ── Input document ───────────────────────────────────────────────────
String document =
    "Alan Turing was a British mathematician. "
  + "He proposed the Turing test in 1950.\n\n"
  + "His work laid the foundation for computer science.";

// ── Pipeline ─────────────────────────────────────────────────────────
PorterStemmer stemmer = tlStemmer.get();

for (String paragraph : paragraphSplitter.split(document)) {
    for (String sentence : sentenceSplitter.split(paragraph)) {
        String[] tokens = tokenizer.split(sentence);
        PennTreebankPOS[] tags = tagger.tag(tokens);

        for (int i = 0; i < tokens.length; i++) {
            if (tags[i].open) { // content word
                String stem = stemmer.stem(tokens[i].toLowerCase());
                System.out.printf("%-20s %-6s %s%n", tokens[i], tags[i], stem);
            }
        }
        System.out.println();
    }
}

// ── Word tokenizers ──────────────────────────────────────────────────
Tokenizer simple    = new SimpleTokenizer();                    // thread-safe
Tokenizer ptb       = PennTreebankTokenizer.getInstance();      // singleton, thread-safe
Tokenizer biTok     = new BreakIteratorTokenizer();             // per-thread
Tokenizer biTokFr   = new BreakIteratorTokenizer(Locale.FRENCH);// locale-aware

String[] tokens = simple.split("He won't go.");
// [He, will, not, go, .]

// ── Sentence splitters ───────────────────────────────────────────────
SentenceSplitter ss  = SimpleSentenceSplitter.getInstance();    // singleton
SentenceSplitter bis = new BreakIteratorSentenceSplitter();     // per-thread
SentenceSplitter bde = new BreakIteratorSentenceSplitter(Locale.GERMAN);

String[] sentences = ss.split("Hello world. How are you?");
// [Hello world., How are you?]

// ── Paragraph splitter ───────────────────────────────────────────────
ParagraphSplitter ps = SimpleParagraphSplitter.getInstance();   // singleton
String[] paragraphs  = ps.split("Para one.\n\nPara two.");
// [Para one., Para two.]

// ── As Function in streams ───────────────────────────────────────────
String[][] allTokens = Arrays.stream(sentences)
        .map(simple)                    // Tokenizer IS a Function<String,String[]>
        .toArray(String[][]::new);