// build.gradle.kts
dependencies {
    implementation("com.github.haifengl:smile-nlp:4.x.x")
}

import smile.nlp.*;
import smile.nlp.normalizer.SimpleNormalizer;
import smile.nlp.tokenizer.*;
import smile.nlp.stemmer.PorterStemmer;
import smile.nlp.pos.*;

// 1. Normalize raw text
SimpleNormalizer normalizer = SimpleNormalizer.getInstance();
String text = normalizer.normalize(rawText);

// 2. Split into paragraphs → sentences → tokens
ParagraphSplitter paragraphs = SimpleParagraphSplitter.getInstance();
SentenceSplitter  sentences  = SimpleSentenceSplitter.getInstance();
Tokenizer         tokenizer  = new SimpleTokenizer();

// 3. POS-tag every sentence (one Porter stemmer per thread)
HMMPOSTagger tagger = HMMPOSTagger.getDefault();
ThreadLocal<PorterStemmer> tlStemmer = ThreadLocal.withInitial(PorterStemmer::new);

for (String para : paragraphs.split(text)) {
    for (String sent : sentences.split(para)) {
        String[] tokens = tokenizer.split(sent);
        PennTreebankPOS[] tags = tagger.tag(tokens);
        PorterStemmer stemmer = tlStemmer.get();

        for (int i = 0; i < tokens.length; i++) {
            if (tags[i].open) {                   // content words only
                String stem = stemmer.stem(tokens[i].toLowerCase());
                System.out.printf("%-20s %-6s %s%n", tokens[i], tags[i], stem);
            }
        }
    }
}

Text doc = Text.of("Machine learning is a subfield of AI.");
Text titled = Text.of("Overview", "Machine learning is a subfield of AI.");
System.out.println(doc.content());   // "Machine learning is a subfield of AI."
System.out.println(titled.title());  // "Overview"

Bigram b = new Bigram("machine", "learning");           // lookup key
NGram  n = new NGram(new String[]{"deep","learning"});  // lookup key

SimpleCorpus corpus = new SimpleCorpus();
corpus.add(corpus.doc("SMILE is a fast machine learning library."));
corpus.add(corpus.doc("Deep learning uses neural networks."));

System.out.println(corpus.docCount());   // 2
System.out.println(corpus.size());       // total tokens
System.out.println(corpus.count("learning")); // term frequency

import smile.nlp.Word2Vec;
import java.nio.file.Path;

// Load Word2vec binary model
Word2Vec w2v = Word2Vec.of(Path.of("GoogleNews-vectors-negative300.bin"));

// Load GloVe text model
Word2Vec glove = Word2Vec.glove(Path.of("glove.6B.100d.txt"));

// Look up a vector
float[] vec = w2v.apply("king");    // throws if not found
w2v.lookup("king").ifPresent(v -> System.out.println(v.length)); // 300

// Cosine similarity
w2v.similarity("king", "queen").ifPresent(System.out::println);  // ≈ 0.65

System.out.println(w2v.size());       // vocabulary size
System.out.println(w2v.dimension());  // embedding dimensionality

import smile.nlp.normalizer.SimpleNormalizer;

SimpleNormalizer norm = SimpleNormalizer.getInstance();   // thread-safe singleton
String clean = norm.normalize("\u201CHello,\u201D   world\u2019s\u2026");
// → "\"Hello,\"   world's..."

import smile.nlp.stemmer.*;

PorterStemmer   porter   = new PorterStemmer();          // one per thread
LancasterStemmer lancas  = new LancasterStemmer();        // shareable
LancasterStemmer withPfx = new LancasterStemmer(true);    // strip prefixes

System.out.println(porter.stem("generalization")); // "gener"
System.out.println(lancas.stem("presumably"));     // "presum"

import smile.nlp.pos.*;

HMMPOSTagger tagger = HMMPOSTagger.getDefault();   // pre-trained HMM model

String[] tokens = {"The", "cat", "sat", "on", "the", "mat", "."};
PennTreebankPOS[] tags = tagger.tag(tokens);

for (int i = 0; i < tokens.length; i++) {
    System.out.printf("%-12s %s%n", tokens[i], tags[i]);
}
// The          DT
// cat          NN
// sat          VBD
// ...

import smile.nlp.dictionary.EnglishStopWords;

EnglishStopWords stopWords = EnglishStopWords.DEFAULT;
System.out.println(stopWords.contains("the"));    // true
System.out.println(stopWords.contains("machine")); // false

import smile.nlp.dictionary.EnglishPunctuations;

EnglishPunctuations punct = EnglishPunctuations.getInstance();
System.out.println(punct.contains("."));   // true
System.out.println(punct.contains("NLP")); // false

import smile.nlp.dictionary.EnglishDictionary;

System.out.println(EnglishDictionary.contains("serendipity")); // true
System.out.println(EnglishDictionary.contains("xyzzy"));       // false

SimpleCorpus corpus = new SimpleCorpus();
// ... load documents via corpus.add(corpus.doc(text)) ...

// Top-20 bigrams by likelihood-ratio score (min raw count > 3)
List<Bigram> top20 = corpus.bigrams(20, 3);

// All bigrams significant at p < 0.001
List<Bigram> sig = corpus.bigrams(0.001, 3);

import smile.nlp.NGram;

// sentences = List<String[]>  (already tokenised)
NGram[][] result = NGram.apriori(sentences, 4, 4);  // up to 4-grams, min freq 4

NGram[] bigrams  = result[2];   // sorted desc by count
NGram[] trigrams = result[3];

import smile.nlp.relevance.*;

// TF-IDF search
RelevanceRanker tfidf = new TFIDF();
corpus.search(tfidf, "machine learning").forEachRemaining(r ->
    System.out.printf("%.4f  %s%n", r.relevance(), r.text().title()));

// BM25 search (Okapi weighting, state-of-the-art)
RelevanceRanker bm25 = new BM25();
corpus.search(bm25, new String[]{"deep", "learning"}).forEachRemaining(r ->
    System.out.printf("%.4f  %s%n", r.relevance(), r.text().title()));

import smile.nlp.taxonomy.*;

// Build a simple IS-A hierarchy
Taxonomy taxonomy = new Taxonomy("entity");
Concept entity  = taxonomy.getConcept("entity");
Concept animal  = entity.addChild("animal");
Concept plant   = entity.addChild("plant");
Concept dog     = animal.addChild("dog");
Concept cat     = animal.addChild("cat");
Concept rose    = plant.addChild("rose");

// Retrieve a concept
Concept c = taxonomy.getConcept("dog");
System.out.println(c.getKeywords());   // [dog]

// Taxonomic distance between two concepts
TaxonomicDistance dist = new TaxonomicDistance(taxonomy);
System.out.println(dist.d("dog", "cat"));   // sibling distance
System.out.println(dist.d("dog", "rose"));  // cross-branch distance

smile.nlp
├── Text, Corpus, SimpleCorpus          – document model & in-memory corpus
├── Bigram, NGram                        – collocation data records
├── Word2Vec                             – word embeddings (Word2vec / GloVe)
├── AnchorText                           – web anchor-text record
│
├── normalizer/
│   ├── Normalizer                       – interface
│   └── SimpleNormalizer                 – Unicode NFKC + whitespace cleanup
│
├── tokenizer/
│   ├── ParagraphSplitter / SimpleParagraphSplitter
│   ├── SentenceSplitter  / SimpleSentenceSplitter / BreakIteratorSentenceSplitter
│   └── Tokenizer         / SimpleTokenizer / PennTreebankTokenizer / BreakIteratorTokenizer
│
├── stemmer/
│   ├── Stemmer                          – interface (extends Function<String,String>)
│   ├── PorterStemmer                    – Porter 1980
│   └── LancasterStemmer                 – Paice/Husk 1990
│
├── pos/
│   ├── PennTreebankPOS                  – 45-tag enum
│   ├── POSTagger                        – interface
│   ├── HMMPOSTagger                     – pre-trained HMM tagger
│   ├── RegexPOSTagger                   – rule-based pre-classifier
│   └── EnglishPOSLexicon                – ~200 k word POS dictionary
│
├── dictionary/
│   ├── StopWords / EnglishStopWords     – stop-word lists
│   ├── Punctuations / EnglishPunctuations
│   ├── Dictionary / EnglishDictionary   – broad-coverage word list
│   └── Abbreviations                    – common abbreviation list
│
├── relevance/
│   ├── RelevanceRanker                  – interface
│   ├── Relevance                        – (document, score) record
│   ├── TFIDF                            – TF-IDF ranker
│   └── BM25                             – Okapi BM25 ranker
│
└── taxonomy/
    ├── Concept                          – tree node with keywords
    ├── Taxonomy                         – concept tree
    └── TaxonomicDistance                – Wu-Palmer semantic distance