baml_language/rig_tests/README.md
E2E test crates for BAML code generation. Each language+fixture combination gets its own test crate.
# Generate all test crates from templates
cargo run -p tools_rig
# Run tests for a specific fixture
cargo test -p python_empty
# Check if crates are in sync with templates
cargo run -p tools_rig -- --check
Edit crates/baml_codegen_tests/src/builders.rs:
define_fixtures! {
empty => {
ObjectPool::empty()
},
your_new_fixture => {
ObjectPool::new()
.with_class(Class { ... })
.with_enum(Enum { ... })
},
}
cargo run -p tools_rig
This creates test crates for all languages:
rig_tests/crates/python_your_new_fixture/rig_tests/crates/typescript_your_new_fixture/ (when TypeScript is added)vim rig_tests/crates/python_your_new_fixture/customizable/test_main.py
Files in customizable/ are preserved across regenerations.
cargo test -p python_your_new_fixture
mkdir -p rig_tests/crate_templates/your_language/{src,customizable}
You need 4 templates with {{fixture_name}} placeholders:
Cargo.toml.template[package]
name = "rig_your_language_{{fixture_name}}"
version.workspace = true
edition.workspace = true
[dependencies]
baml_codegen_tests = { path = "../../../crates/baml_codegen_tests" }
baml_codegen_your_language = { path = "../../../crates/baml_codegen_your_language" }
[[test]]
name = "generated_code"
path = "src/lib.rs"
harness = true
build.rs.templateuse std::env;
use std::fs;
use std::path::PathBuf;
fn main() {
let manifest_dir = PathBuf::from(env::var("CARGO_MANIFEST_DIR").unwrap());
let generated_dir = manifest_dir.join("generated");
// Clean and recreate
if generated_dir.exists() {
fs::remove_dir_all(&generated_dir).unwrap();
}
fs::create_dir_all(&generated_dir).unwrap();
// Generate code
let fixture = baml_codegen_tests::fixtures::{{fixture_name}}();
let output = baml_codegen_your_language::to_source_code(&fixture, &PathBuf::from("."));
// Write generated files
for (path, content) in output {
let file_path = generated_dir.join(&path);
if let Some(parent) = file_path.parent() {
fs::create_dir_all(parent).unwrap();
}
fs::write(&file_path, &content).unwrap();
}
// Symlink customizable files
let customizable_dir = manifest_dir.join("customizable");
if customizable_dir.exists() {
for entry in fs::read_dir(&customizable_dir).unwrap() {
let entry = entry.unwrap();
let src = entry.path();
if !src.is_file() { continue; }
let dst = generated_dir.join(entry.file_name());
if dst.exists() || dst.symlink_metadata().is_ok() {
let _ = fs::remove_file(&dst);
}
#[cfg(unix)]
std::os::unix::fs::symlink(&src, &dst).unwrap();
#[cfg(windows)]
std::os::windows::fs::symlink_file(&src, &dst).unwrap();
}
}
// Write test script (language-specific)
let test_sh = r#"#!/usr/bin/env bash
set -e
cd "$(dirname "$0")"
# Add your language-specific test commands here
"#;
fs::write(generated_dir.join("test.sh"), test_sh).unwrap();
#[cfg(unix)]
{
use std::os::unix::fs::PermissionsExt;
let mut perms = fs::metadata(generated_dir.join("test.sh")).unwrap().permissions();
perms.set_mode(0o755);
fs::set_permissions(generated_dir.join("test.sh"), perms).unwrap();
}
println!("cargo:rerun-if-changed=build.rs");
// Watch customizable files
if customizable_dir.exists() {
for entry in fs::read_dir(&customizable_dir).unwrap() {
let entry = entry.unwrap();
if entry.path().is_file() {
println!("cargo:rerun-if-changed={}", entry.path().display());
}
}
}
}
src/lib.rs.templateuse std::path::PathBuf;
use std::process::Command;
fn generated_dir() -> PathBuf {
PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("generated")
}
#[test]
fn test_generated_code() {
let dir = generated_dir();
let test_script = dir.join("test.sh");
assert!(test_script.exists(), "test.sh not found");
let output = Command::new("bash")
.arg(&test_script)
.current_dir(&dir)
.output()
.expect("Failed to run test.sh");
assert!(
output.status.success(),
"test.sh failed:\nstdout: {}\nstderr: {}",
String::from_utf8_lossy(&output.stdout),
String::from_utf8_lossy(&output.stderr)
);
}
customizable/test_main.* (language-specific)For Python (test_main.py.template):
#!/usr/bin/env python3
"""Pytest tests for {{fixture_name}} fixture."""
def test_imports():
"""Test that baml_client can be imported."""
import baml_client # noqa: F401
def test_fixture_specific():
"""Fixture-specific tests for {{fixture_name}}."""
# TODO: Add fixture-specific tests here
pass
cargo run -p tools_rig
Creates <language>_<fixture> crates for all fixtures.
cargo test -p your_language_*
rig_tests/
├── crate_templates/ # Templates (NOT in Cargo workspace)
│ └── <language>/
│ ├── Cargo.toml.template
│ ├── build.rs.template
│ ├── src/lib.rs.template
│ └── customizable/*.template
│
└── crates/ # Generated crates (IN workspace)
└── <language>_<fixture>/
├── Cargo.toml # Always regenerated
├── build.rs # Always regenerated
├── src/lib.rs # Always regenerated
├── customizable/ # Preserved across regenerations
└── generated/ # Build output (gitignored)
crate_templates/<language>/ define structuretools_rig generates crates by replacing {{fixture_name}}build.rs (at compile time):
baml_codegen_<language>::to_source_code(fixture)generated/customizable/* into generated/test.shlib.rs runs test.sh via bashtest.sh runs language-specific checksAlways regenerated:
Cargo.toml, build.rs, src/lib.rsPreserved (only created if missing):
customizable/*cargo run -p tools_rig -- --check
Fails if crates are out of sync with templates.