import smile.nlp.Text;

// body-only (title is "", id is a random UUID)
Text doc = Text.of("The quick brown fox jumps over the lazy dog.");

// with title
Text doc = Text.of("Fable", "The quick brown fox jumps over the lazy dog.");

public record Bigram(String w1, String w2, int count, double score)
        implements Comparable<Bigram> { … }

import smile.nlp.Bigram;

// Lookup key (count/score unknown)
Bigram key = new Bigram("machine", "learning");
System.out.println(key);          // (machine learning)

// With statistics (returned by SimpleCorpus.bigrams)
Bigram b = new Bigram("machine", "learning", 42, 312.7);
System.out.println(b);            // (machine learning, 42, 312.70)
System.out.println(b.w1());       // machine
System.out.println(b.score());    // 312.7

public record NGram(String[] words, int count) implements Comparable<NGram> { … }

import smile.nlp.NGram;

// Lookup key (count unknown)
NGram key = new NGram(new String[]{"New", "York", "City"});
System.out.println(key);           // [New, York, City]

// With frequency (returned by NGram.apriori)
NGram ng  = new NGram(new String[]{"machine", "learning"}, 17);
System.out.println(ng);            // ([machine, learning], 17)
System.out.println(ng.words()[0]); // machine

import smile.nlp.*;
import smile.nlp.tokenizer.*;
import smile.nlp.dictionary.*;

// Default corpus (English, stop words and punctuation filtered)
SimpleCorpus corpus = new SimpleCorpus();

// Custom components
SimpleCorpus corpus = new SimpleCorpus(
    SimpleSentenceSplitter.getInstance(),
    new SimpleTokenizer(),
    EnglishStopWords.COMPREHENSIVE,
    EnglishPunctuations.getInstance()
);

// doc() applies the corpus's tokenizer, stop-word filter, etc.
corpus.add(corpus.doc("SMILE is a fast, general machine learning library."));
corpus.add(corpus.doc("Machine learning powers modern NLP applications."));
corpus.add(corpus.doc("Deep learning is a subfield of machine learning."));

import smile.nlp.normalizer.SimpleNormalizer;

SimpleNormalizer normalizer = SimpleNormalizer.getInstance();
Files.lines(Path.of("data/corpus.txt"))
     .map(normalizer::normalize)
     .filter(line -> !line.isBlank())
     .forEach(line -> corpus.add(corpus.doc(line)));

System.out.println("Documents : " + corpus.docCount());
System.out.println("Tokens    : " + corpus.size());
System.out.println("Unique    : " + corpus.termCount());
System.out.println("Bigrams   : " + corpus.bigramCount());
System.out.println("Avg size  : " + corpus.avgDocSize());

List<Bigram> bigrams(int k, int minFrequency)

SimpleCorpus corpus = new SimpleCorpus();
// ... load documents ...

// Top 10 bigrams appearing more than 5 times
List<Bigram> top10 = corpus.bigrams(10, 5);

for (Bigram b : top10) {
    System.out.printf("%-20s  count=%-5d  score=%.2f%n",
        b.w1() + " " + b.w2(), b.count(), b.score());
}

new york              count=46     score=545.16
carbon dioxide        count=38     score=412.83
european union        count=31     score=298.74

List<Bigram> bigrams(double p, int minFrequency)

// All bigrams significant at the 0.01% level
List<Bigram> significant = corpus.bigrams(0.0001, 5);
System.out.println("Significant bigrams: " + significant.size());

public static NGram[][] apriori(Collection<String[]> sentences,
                                int maxNGramSize,
                                int minFrequency)

import smile.nlp.NGram;
import smile.nlp.stemmer.PorterStemmer;
import smile.nlp.tokenizer.*;

PorterStemmer stemmer = new PorterStemmer();
SimpleTokenizer tokenizer = new SimpleTokenizer();
List<String[]> sentences = new ArrayList<>();

String text = Files.readString(Path.of("data/article.txt"));
for (String para : SimpleParagraphSplitter.getInstance().split(text)) {
    for (String sent : SimpleSentenceSplitter.getInstance().split(para)) {
        String[] tokens = tokenizer.split(sent);
        // lowercase + stem for better conflation
        for (int i = 0; i < tokens.length; i++) {
            tokens[i] = stemmer.stem(tokens[i].toLowerCase());
        }
        sentences.add(tokens);
    }
}

// Extract up to 4-grams appearing at least 4 times
NGram[][] result = NGram.apriori(sentences, 4, 4);

// result[0] → empty  (no such thing as 0-grams)
// result[1] → significant unigrams
// result[2] → significant bigrams
// result[3] → significant trigrams
// result[4] → significant 4-grams

for (int n = 1; n < result.length; n++) {
    System.out.println("=== " + n + "-grams ===");
    for (NGram ng : result[n]) {
        System.out.println(ng);   // ([word1, word2], count)
    }
}

=== 1-grams ===
([machin], 42)
([learn], 39)
([algorithm], 28)
…
=== 2-grams ===
([machin, learn], 17)
([neural, network], 12)
…
=== 3-grams ===
([deep, neural, network], 7)
…

import smile.nlp.*;
import smile.nlp.normalizer.SimpleNormalizer;
import smile.nlp.stemmer.PorterStemmer;
import smile.nlp.tokenizer.*;

import java.nio.file.Files;
import java.nio.file.Path;
import java.util.ArrayList;
import java.util.List;

public class CollocationExample {
    public static void main(String[] args) throws Exception {
        SimpleNormalizer normalizer = SimpleNormalizer.getInstance();

        // ── Bigram collocations ──────────────────────────────────────────
        SimpleCorpus corpus = new SimpleCorpus();
        Files.lines(Path.of("data/news.txt"))
             .map(normalizer::normalize)
             .filter(line -> !line.isBlank())
             .forEach(line -> corpus.add(corpus.doc(line)));

        System.out.printf("Corpus: %d documents, %d tokens%n",
                corpus.docCount(), corpus.size());

        // Top 20 bigrams appearing more than 3 times
        System.out.println("\n=== Top 20 Bigram Collocations ===");
        for (Bigram b : corpus.bigrams(20, 3)) {
            System.out.printf("  %-25s  count=%3d  score=%7.2f%n",
                b.w1() + " " + b.w2(), b.count(), b.score());
        }

        // All bigrams significant at p < 0.001
        List<Bigram> sig = corpus.bigrams(0.001, 3);
        System.out.printf("%nBigrams significant at p<0.001: %d%n", sig.size());

        // ── N-gram collocations ──────────────────────────────────────────
        PorterStemmer stemmer = new PorterStemmer();
        SimpleTokenizer tokenizer = new SimpleTokenizer();
        List<String[]> sentences  = new ArrayList<>();

        String text = Files.readString(Path.of("data/article.txt"));
        text = normalizer.normalize(text);

        for (String para : SimpleParagraphSplitter.getInstance().split(text)) {
            for (String sent : SimpleSentenceSplitter.getInstance().split(para)) {
                String[] tokens = tokenizer.split(sent);
                for (int i = 0; i < tokens.length; i++) {
                    tokens[i] = stemmer.stem(tokens[i].toLowerCase());
                }
                sentences.add(tokens);
            }
        }

        System.out.println("\n=== N-Gram Collocations (max 4-gram, min freq 3) ===");
        NGram[][] ngrams = NGram.apriori(sentences, 4, 3);
        for (int n = 1; n < ngrams.length; n++) {
            System.out.printf("%n--- %d-grams: %d ---%n", n, ngrams[n].length);
            int limit = Math.min(5, ngrams[n].length);
            for (int i = 0; i < limit; i++) {
                System.out.println("  " + ngrams[n][i]);
            }
        }
    }
}

tokens[i] = stemmer.stem(tokens[i].toLowerCase());

// ── Bigram ───────────────────────────────────────────────────────────
Bigram key  = new Bigram("machine", "learning");          // lookup key
Bigram full = new Bigram("machine", "learning", 42, 312.7); // with stats
String w1   = full.w1();      // "machine"
int count   = full.count();   // 42
double score = full.score();  // 312.7

// ── NGram ────────────────────────────────────────────────────────────
NGram ng    = new NGram(new String[]{"machine","learning"}, 17);
String[] ws = ng.words();     // ["machine", "learning"]
int freq    = ng.count();     // 17

// ── SimpleCorpus bigram collocations ─────────────────────────────────
SimpleCorpus corpus = new SimpleCorpus();
corpus.add(corpus.doc("Some text goes here."));

List<Bigram> top20 = corpus.bigrams(20, 3);          // top-k
List<Bigram> sig   = corpus.bigrams(0.001, 3);       // by p-value

// ── NGram.apriori ────────────────────────────────────────────────────
NGram[][] result = NGram.apriori(sentences, 4, 4);   // up to 4-grams
NGram[] bigrams  = result[2];                         // bigrams
NGram[] trigrams = result[3];                         // trigrams