1
#![allow(clippy::expect_used)]
2
use git_version as _;
3

            
4
use conjure_cp::defaults::DEFAULT_RULE_SETS;
5
use conjure_cp::parse::tree_sitter::parse_essence_file_native;
6
use conjure_cp::rule_engine::{rewrite_morph, rewrite_naive};
7
use conjure_cp::solver::Solver;
8
use conjure_cp::solver::adaptors::*;
9
use conjure_cp_cli::utils::testing::{normalize_solutions_for_comparison, read_default_rule_trace};
10
use std::collections::{BTreeMap, BTreeSet};
11
use std::env;
12
use std::error::Error;
13
use std::fs;
14
use std::fs::File;
15
use std::path::Path;
16
use tracing_subscriber::{Layer, filter::EnvFilter, filter::FilterFn, fmt, layer::SubscriberExt};
17

            
18
use std::sync::Arc;
19
use std::sync::RwLock;
20

            
21
use conjure_cp::ast::{Literal, Name};
22
use conjure_cp::context::Context;
23
use conjure_cp::parse::tree_sitter::parse_essence_file;
24
use conjure_cp::rule_engine::resolve_rule_sets;
25
use conjure_cp::settings::{
26
    Parser, QuantifiedExpander, Rewriter, SolverFamily, set_comprehension_expander,
27
    set_current_parser, set_current_rewriter, set_current_solver_family,
28
    set_default_rule_trace_enabled, set_minion_discrete_threshold,
29
    set_rule_trace_aggregates_enabled, set_rule_trace_enabled, set_rule_trace_verbose_enabled,
30
};
31
use conjure_cp_cli::utils::conjure::solutions_to_json;
32
use conjure_cp_cli::utils::conjure::{get_solutions, get_solutions_from_conjure};
33
use conjure_cp_cli::utils::testing::save_stats_json;
34
use conjure_cp_cli::utils::testing::{read_solutions_json, save_solutions_json};
35
#[allow(clippy::single_component_path_imports, unused_imports)]
36
use conjure_cp_rules;
37
use pretty_assertions::assert_eq;
38
use tests_integration::TestConfig;
39
use tests_integration::golden_files::assert_no_redundant_expected_files;
40

            
41
#[derive(Clone, Copy, Debug)]
42
struct RunCase<'a> {
43
    parser: Parser,
44
    rewriter: Rewriter,
45
    comprehension_expander: QuantifiedExpander,
46
    solver: SolverFamily,
47
    case_name: &'a str,
48
}
49

            
50
2251
fn run_case_label(
51
2251
    path: &str,
52
2251
    essence_base: &str,
53
2251
    extension: &str,
54
2251
    run_case: RunCase<'_>,
55
2251
) -> String {
56
2251
    format!(
57
        "test_dir={path}, model={essence_base}.{extension}, parser={}, rewriter={}, comprehension_expander={}, solver={}",
58
        run_case.parser,
59
        run_case.rewriter,
60
        run_case.comprehension_expander,
61
2251
        run_case.solver.as_str()
62
    )
63
2251
}
64

            
65
295
fn integration_test(path: &str, essence_base: &str, extension: &str) -> Result<(), Box<dyn Error>> {
66
295
    let accept = env::var("ACCEPT").unwrap_or("false".to_string()) == "true";
67

            
68
295
    if accept {
69
        clean_test_dir_for_accept(path, essence_base, extension)?;
70
295
    }
71

            
72
295
    let file_config: TestConfig =
73
295
        if let Ok(config_contents) = fs::read_to_string(format!("{path}/config.toml")) {
74
295
            toml::from_str(&config_contents).unwrap()
75
        } else {
76
            Default::default()
77
        };
78

            
79
295
    let config = file_config;
80

            
81
295
    let validate_with_conjure = config.validate_with_conjure;
82
295
    let minion_discrete_threshold = config.minion_discrete_threshold;
83

            
84
295
    let parsers = config
85
295
        .configured_parsers()
86
295
        .map_err(|err| std::io::Error::new(std::io::ErrorKind::InvalidInput, err))?;
87
295
    let rewriters = config
88
295
        .configured_rewriters()
89
295
        .map_err(|err| std::io::Error::new(std::io::ErrorKind::InvalidInput, err))?;
90
295
    let comprehension_expanders = config
91
295
        .configured_comprehension_expanders()
92
295
        .map_err(|err| std::io::Error::new(std::io::ErrorKind::InvalidInput, err))?;
93
295
    let solvers = config
94
295
        .configured_solvers()
95
295
        .map_err(|err| std::io::Error::new(std::io::ErrorKind::InvalidInput, err))?;
96

            
97
    // Conjure output depends only on the input model, so cache it once per test case.
98
295
    let model_path = format!("{path}/{essence_base}.{extension}");
99
295
    let conjure_solutions = if accept && validate_with_conjure {
100
        eprintln!("[integration] loading Conjure reference solutions for {model_path}");
101
        Some(Arc::new(
102
            get_solutions_from_conjure(&model_path, None, Default::default()).map_err(|err| {
103
                std::io::Error::other(format!(
104
                    "failed to fetch Conjure reference solutions for {model_path}: {err}"
105
                ))
106
            })?,
107
        ))
108
    } else {
109
295
        if accept && !validate_with_conjure {
110
            eprintln!("[integration] skipping Conjure validation for {model_path}");
111
295
        }
112
295
        None
113
    };
114
295
    let mut allowed_expected_files = BTreeSet::new();
115

            
116
573
    for parser in parsers.iter().copied() {
117
1135
        for rewriter in rewriters.clone() {
118
1261
            for comprehension_expander in comprehension_expanders.clone() {
119
2251
                for solver in solvers.clone() {
120
2251
                    let case_name = run_case_name(parser, rewriter, comprehension_expander);
121
2251
                    let run_case = RunCase {
122
2251
                        parser,
123
2251
                        rewriter,
124
2251
                        comprehension_expander,
125
2251
                        solver,
126
2251
                        case_name: case_name.as_str(),
127
2251
                    };
128
2251
                    let file = File::create(format!(
129
                        "{path}/{}-{}-generated-rule-trace.txt",
130
                        run_case.case_name,
131
2251
                        run_case.solver.as_str()
132
                    ))?;
133
2251
                    let subscriber = Arc::new(
134
2251
                        tracing_subscriber::registry().with(
135
2251
                            fmt::layer()
136
2251
                                .with_writer(file)
137
2251
                                .with_level(false)
138
2251
                                .without_time()
139
2251
                                .with_target(false)
140
2251
                                .with_filter(EnvFilter::new("rule_engine_rule_trace=trace"))
141
350130
                                .with_filter(FilterFn::new(|meta| {
142
350130
                                    meta.target() == "rule_engine_rule_trace"
143
350130
                                })),
144
                        ),
145
                    )
146
2251
                        as Arc<dyn tracing::Subscriber + Send + Sync>;
147
2251
                    let run_label = run_case_label(path, essence_base, extension, run_case);
148
2251
                    eprintln!("[integration] running {run_label}");
149
2251
                    let default_rule_trace_enabled = matches!(rewriter, Rewriter::Naive);
150
2251
                    set_rule_trace_enabled(true);
151
2251
                    set_default_rule_trace_enabled(default_rule_trace_enabled);
152
2251
                    set_rule_trace_verbose_enabled(false);
153
2251
                    set_rule_trace_aggregates_enabled(false);
154
2251
                    tracing::subscriber::with_default(subscriber, || {
155
2251
                        integration_test_inner(
156
2251
                            path,
157
2251
                            essence_base,
158
2251
                            extension,
159
2251
                            run_case,
160
2251
                            minion_discrete_threshold,
161
2251
                            conjure_solutions.clone(),
162
2251
                            accept,
163
                        )
164
2251
                    })
165
2251
                    .map_err(|err| std::io::Error::other(format!("{run_label}: {err}")))?;
166
2251
                    allowed_expected_files.extend(expected_integration_files_for_case(
167
2251
                        run_case.case_name,
168
2251
                        solver,
169
                    ));
170
                }
171
            }
172
        }
173
    }
174

            
175
295
    assert_no_redundant_expected_files(Path::new(path), &allowed_expected_files, None)?;
176

            
177
295
    Ok(())
178
295
}
179

            
180
/// Runs an integration test for a given Conjure model by:
181
/// 1. Parsing the model from an Essence file.
182
/// 2. Rewriting the model according to predefined rule sets.
183
/// 3. Solving the model using the Minion solver and validating the solutions.
184
/// 4. Comparing generated rule traces with expected outputs.
185
///
186
/// This function operates in multiple stages:
187
///
188
/// - **Parsing Stage**
189
///   - **Stage 1a (Default)**: Reads the Essence model file and verifies that it parses correctly using `parser = "tree-sitter"`.
190
///   - **Stage 1b (Optional)**: Reads the Essence model file and verifies that it parses correctly using `parser = "via-conjure"`.
191
///
192
/// - **Rewrite Stage**
193
///   - **Stage 2a**: Applies a set of rules to the parsed model and validates the result.
194
///
195
/// - **Solution Stage**
196
///   - **Stage 3a (Default)**: Uses Minion to solve the model and save the solutions.
197
///   - **Stage 3b (ACCEPT only)**: Compares solutions against Conjure-generated solutions.
198
///
199
/// - **Rule Trace Validation Stage**
200
///   - **Stage 4a (Default)**: Checks that the generated rules match expected traces.
201
///
202
/// # Arguments
203
///
204
/// * `path` - The file path where the Essence model and other resources are located.
205
/// * `essence_base` - The base name of the Essence model file.
206
/// * `extension` - The file extension for the Essence model.
207
///
208
/// # Errors
209
///
210
/// Returns an error if any stage fails due to a mismatch with expected results or file I/O issues.
211
#[allow(clippy::unwrap_used)]
212
2251
fn integration_test_inner(
213
2251
    path: &str,
214
2251
    essence_base: &str,
215
2251
    extension: &str,
216
2251
    run_case: RunCase<'_>,
217
2251
    minion_discrete_threshold: usize,
218
2251
    conjure_solutions: Option<Arc<Vec<BTreeMap<Name, Literal>>>>,
219
2251
    accept: bool,
220
2251
) -> Result<(), Box<dyn Error>> {
221
2251
    let parser = run_case.parser;
222
2251
    let rewriter = run_case.rewriter;
223
2251
    let comprehension_expander = run_case.comprehension_expander;
224
2251
    let solver_fam = run_case.solver;
225
2251
    let case_name = run_case.case_name;
226

            
227
2251
    let context: Arc<RwLock<Context<'static>>> = Default::default();
228

            
229
2251
    set_current_parser(parser);
230
2251
    set_current_rewriter(rewriter);
231
2251
    set_comprehension_expander(comprehension_expander);
232
2251
    set_current_solver_family(solver_fam);
233
2251
    set_minion_discrete_threshold(minion_discrete_threshold);
234

            
235
    // File path
236
2251
    let file_path = format!("{path}/{essence_base}.{extension}");
237

            
238
    // Stage 1a/1b: Parse the model using the selected parser.
239
2251
    let parsed_model = match parser {
240
        Parser::TreeSitter => {
241
1339
            let mut ctx = context.as_ref().write().unwrap();
242
1339
            ctx.essence_file_name = Some(format!("{path}/{essence_base}.{extension}"));
243
1339
            parse_essence_file_native(&file_path, context.clone())?
244
        }
245
912
        Parser::ViaConjure => parse_essence_file(&file_path, context.clone())?,
246
    };
247
    // Stage 2a: Rewrite the model using the rule engine
248
2251
    let mut extra_rules = vec![];
249

            
250
2251
    if let SolverFamily::Sat(sat_encoding) = solver_fam {
251
690
        extra_rules.push(sat_encoding.as_rule_set());
252
1561
    }
253

            
254
2251
    let mut rules_to_load = DEFAULT_RULE_SETS.to_vec();
255
2251
    rules_to_load.extend(extra_rules);
256

            
257
2251
    let rule_sets = resolve_rule_sets(solver_fam, &rules_to_load)?;
258

            
259
2251
    let model = parsed_model;
260

            
261
2251
    let rewritten_model = match rewriter {
262
1141
        Rewriter::Naive => rewrite_naive(&model, &rule_sets, false)?,
263
1110
        Rewriter::Morph(config) => rewrite_morph(model, &rule_sets, false, config),
264
    };
265
2251
    let solver_input_file = None;
266
2251
    let solver = match solver_fam {
267
1085
        SolverFamily::Minion => Solver::new(Minion::default()),
268
690
        SolverFamily::Sat(_) => Solver::new(Sat::default()),
269

            
270
476
        SolverFamily::Smt(_) => Solver::new(Smt::default()),
271
    };
272

            
273
2251
    let solutions = {
274
2251
        let solved = get_solutions(solver, rewritten_model, 0, &solver_input_file, false)?;
275
2251
        save_solutions_json(&solved, path, case_name, solver_fam)?;
276
2251
        solved
277
    };
278

            
279
    // Stage 3b: Check solutions against Conjure when ACCEPT=true and validation is enabled.
280
2251
    if accept && conjure_solutions.is_some() {
281
        let conjure_solutions = conjure_solutions
282
            .as_deref()
283
            .expect("conjure solutions should be present when Conjure validation is enabled");
284

            
285
        let username_solutions = normalize_solutions_for_comparison(&solutions);
286
        let conjure_solutions = normalize_solutions_for_comparison(conjure_solutions);
287

            
288
        let mut conjure_solutions_json = solutions_to_json(&conjure_solutions);
289
        let mut username_solutions_json = solutions_to_json(&username_solutions);
290

            
291
        conjure_solutions_json.sort_all_objects();
292
        username_solutions_json.sort_all_objects();
293

            
294
        assert_eq!(
295
            username_solutions_json, conjure_solutions_json,
296
            "Solutions (<) do not match conjure (>)!"
297
        );
298
2251
    }
299

            
300
    // When ACCEPT=true, copy all generated files to expected
301
2251
    if accept {
302
        // Always overwrite these ones. Unlike the rest, we don't need to selectively do these
303
        // based on the test results, so they don't get done later.
304
        copy_generated_to_expected(path, case_name, "solutions", "json", solver_fam)?;
305
        copy_human_trace_generated_to_expected(path, case_name, solver_fam)?;
306
2251
    }
307

            
308
    // Check Stage 3a (solutions)
309
2251
    let expected_solutions_json = read_solutions_json(path, case_name, "expected", solver_fam)?;
310
2251
    let username_solutions_json = solutions_to_json(&solutions);
311
2251
    assert_eq!(username_solutions_json, expected_solutions_json);
312

            
313
    // TODO: Implement rule trace validation for morph
314
2251
    match rewriter {
315
1110
        Rewriter::Morph(_) => {}
316
        Rewriter::Naive => {
317
1141
            let generated = read_default_rule_trace(path, case_name, "generated", &solver_fam)?;
318
1141
            let expected = read_default_rule_trace(path, case_name, "expected", &solver_fam)?;
319

            
320
1141
            assert_eq!(
321
                expected, generated,
322
                "Generated rule trace does not match the expected trace!"
323
            );
324
        }
325
    }
326

            
327
2251
    save_stats_json(context, path, case_name, solver_fam)?;
328

            
329
2251
    Ok(())
330
2251
}
331

            
332
2251
fn run_case_name(
333
2251
    parser: Parser,
334
2251
    rewriter: Rewriter,
335
2251
    comprehension_expander: QuantifiedExpander,
336
2251
) -> String {
337
2251
    format!("{parser}-{rewriter}-{comprehension_expander}")
338
2251
}
339

            
340
/// Returns the expected snapshot files for an executed integration run case.
341
2251
fn expected_integration_files_for_case(case_name: &str, solver: SolverFamily) -> BTreeSet<String> {
342
2251
    let solver_name = solver.as_str();
343
2251
    BTreeSet::from([
344
2251
        format!("{case_name}-{solver_name}.expected-solutions.json"),
345
2251
        format!("{case_name}-{solver_name}-expected-rule-trace.txt"),
346
2251
    ])
347
2251
}
348

            
349
fn clean_test_dir_for_accept(
350
    path: &str,
351
    essence_base: &str,
352
    extension: &str,
353
) -> Result<(), std::io::Error> {
354
    let input_filename = format!("{essence_base}.{extension}");
355

            
356
    for entry in std::fs::read_dir(path)? {
357
        let entry = entry?;
358
        let file_name = entry.file_name();
359
        let file_name = file_name.to_string_lossy();
360
        let entry_path = entry.path();
361

            
362
        if file_name == input_filename || file_name == "config.toml" {
363
            continue;
364
        }
365

            
366
        if entry_path.is_dir() {
367
            std::fs::remove_dir_all(entry_path)?;
368
        } else {
369
            std::fs::remove_file(entry_path)?;
370
        }
371
    }
372

            
373
    Ok(())
374
}
375

            
376
fn copy_human_trace_generated_to_expected(
377
    path: &str,
378
    test_name: &str,
379
    solver: SolverFamily,
380
) -> Result<(), std::io::Error> {
381
    let solver_name = solver.as_str();
382

            
383
    std::fs::copy(
384
        format!("{path}/{test_name}-{solver_name}-generated-rule-trace.txt"),
385
        format!("{path}/{test_name}-{solver_name}-expected-rule-trace.txt"),
386
    )?;
387
    Ok(())
388
}
389

            
390
fn copy_generated_to_expected(
391
    path: &str,
392
    test_name: &str,
393
    stage: &str,
394
    extension: &str,
395
    solver: SolverFamily,
396
) -> Result<(), std::io::Error> {
397
    let marker = solver.as_str();
398

            
399
    std::fs::copy(
400
        format!("{path}/{test_name}-{marker}.generated-{stage}.{extension}"),
401
        format!("{path}/{test_name}-{marker}.expected-{stage}.{extension}"),
402
    )?;
403
    Ok(())
404
}
405

            
406
#[test]
407
1
fn assert_conjure_present() {
408
1
    conjure_cp_cli::find_conjure::conjure_executable().unwrap();
409
1
}
410

            
411
include!(concat!(env!("OUT_DIR"), "/gen_tests.rs"));