Grcov report - integration

1

#![allow(clippy::expect_used)]

2

use conjure_cp::bug;

3

use conjure_cp::rule_engine::get_rules_grouped;

4

use git_version as _;

5

6

use conjure_cp::defaults::DEFAULT_RULE_SETS;

7

use conjure_cp::parse::tree_sitter::parse_essence_file_native;

8

use conjure_cp::rule_engine::rewrite_naive;

9

use conjure_cp::solver::Solver;

10

use conjure_cp::solver::adaptors::*;

11

use conjure_cp_cli::utils::testing::{normalize_solutions_for_comparison, read_human_rule_trace};

12

use itertools::Itertools;

13

use std::collections::BTreeMap;

14

use std::env;

15

use std::error::Error;

16

use std::fs;

17

use std::fs::File;

18

use tracing_subscriber::{Layer, filter::EnvFilter, filter::FilterFn, fmt, layer::SubscriberExt};

19

use tree_morph::{helpers::select_panic, prelude::*};

20

21

use std::sync::Arc;

22

use std::sync::RwLock;

23

24

use conjure_cp::ast::{Literal, Name};

25

use conjure_cp::context::Context;

26

use conjure_cp::parse::tree_sitter::parse_essence_file;

27

use conjure_cp::rule_engine::resolve_rule_sets;

28

use conjure_cp::settings::{

29

    Parser, QuantifiedExpander, Rewriter, SolverFamily, set_comprehension_expander,

30

    set_current_parser, set_current_rewriter, set_current_solver_family,

31

    set_minion_discrete_threshold,

32

};

33

use conjure_cp_cli::utils::conjure::solutions_to_json;

34

use conjure_cp_cli::utils::conjure::{get_solutions, get_solutions_from_conjure};

35

use conjure_cp_cli::utils::testing::save_stats_json;

36

use conjure_cp_cli::utils::testing::{read_solutions_json, save_solutions_json};

37

#[allow(clippy::single_component_path_imports, unused_imports)]

38

use conjure_cp_rules;

39

use pretty_assertions::assert_eq;

40

use tests_integration::TestConfig;

41

42

#[derive(Clone, Copy, Debug)]

43

struct RunCase<'a> {

44

    parser: Parser,

45

    rewriter: Rewriter,

46

    comprehension_expander: QuantifiedExpander,

47

    solver: SolverFamily,

48

    case_name: &'a str,

49

50

51

1291

fn run_case_label(

52

1291

    path: &str,

53

1291

    essence_base: &str,

54

1291

    extension: &str,

55

1291

    run_case: RunCase<'_>,

56

1291

) -> String {

57

1291

    format!(

58

        "test_dir={path}, model={essence_base}.{extension}, parser={}, rewriter={}, comprehension_expander={}, solver={}",

59

        run_case.parser,

60

        run_case.rewriter,

61

        run_case.comprehension_expander,

62

1291

        run_case.solver.as_str()

63

64

1291

65

66

854

fn integration_test(path: &str, essence_base: &str, extension: &str) -> Result<(), Box<dyn Error>> {

67

854

    let accept = env::var("ACCEPT").unwrap_or("false".to_string()) == "true";

68

69

854

    if accept {

70

        clean_test_dir_for_accept(path, essence_base, extension)?;

71

854

72

73

854

    let file_config: TestConfig =

74

854

        if let Ok(config_contents) = fs::read_to_string(format!("{path}/config.toml")) {

75

854

            toml::from_str(&config_contents).unwrap()

76

        } else {

77

            Default::default()

78

};

79

80

854

    let config = file_config;

81

82

854

    let validate_with_conjure = config.validate_with_conjure;

83

854

    let minion_discrete_threshold = config.minion_discrete_threshold;

84

85

854

    let parsers = config

86

854

        .configured_parsers()

87

854

        .map_err(|err| std::io::Error::new(std::io::ErrorKind::InvalidInput, err))?;

88

854

    let rewriters = config

89

854

        .configured_rewriters()

90

854

        .map_err(|err| std::io::Error::new(std::io::ErrorKind::InvalidInput, err))?;

91

854

    let comprehension_expanders = config

92

854

        .configured_comprehension_expanders()

93

854

        .map_err(|err| std::io::Error::new(std::io::ErrorKind::InvalidInput, err))?;

94

854

    let solvers = config

95

854

        .configured_solvers()

96

854

        .map_err(|err| std::io::Error::new(std::io::ErrorKind::InvalidInput, err))?;

97

    // Conjure output depends only on the input model, so cache it once per test case.

98

854

    let model_path = format!("{path}/{essence_base}.{extension}");

99

854

    let conjure_solutions = if accept && validate_with_conjure {

100

        eprintln!("[integration] loading Conjure reference solutions for {model_path}");

101

        Some(Arc::new(

102

            get_solutions_from_conjure(&model_path, Default::default()).map_err(|err| {

103

                std::io::Error::other(format!(

104

                    "failed to fetch Conjure reference solutions for {model_path}: {err}"

105

))

106

            })?,

107

))

108

    } else {

109

854

        if accept && !validate_with_conjure {

110

            eprintln!("[integration] skipping Conjure validation for {model_path}");

111

854

112

854

        None

113

};

114

115

914

    for parser in parsers {

116

914

        for rewriter in rewriters.clone() {

117

998

            for comprehension_expander in comprehension_expanders.clone() {

118

1291

                for solver in solvers.clone() {

119

1291

                    let case_name = run_case_name(parser, rewriter, comprehension_expander);

120

1291

                    let run_case = RunCase {

121

1291

                        parser,

122

1291

                        rewriter,

123

1291

                        comprehension_expander,

124

1291

                        solver,

125

1291

                        case_name: case_name.as_str(),

126

1291

};

127

1291

                    let file = File::create(format!(

128

                        "{path}/{}-{}-generated-rule-trace.txt",

129

                        run_case.case_name,

130

1291

                        run_case.solver.as_str()

131

                    ))?;

132

1291

                    let subscriber = Arc::new(

133

1291

                        tracing_subscriber::registry().with(

134

1291

                            fmt::layer()

135

1291

                                .with_writer(file)

136

1291

                                .with_level(false)

137

1291

                                .without_time()

138

1291

                                .with_target(false)

139

1291

                                .with_filter(EnvFilter::new("rule_engine_human=trace"))

140

135813

                                .with_filter(FilterFn::new(|meta| {

141

135813

                                    meta.target() == "rule_engine_human"

142

135813

                                })),

143

),

144

145

1291

                        as Arc<dyn tracing::Subscriber + Send + Sync>;

146

1291

                    let run_label = run_case_label(path, essence_base, extension, run_case);

147

1291

                    eprintln!("[integration] running {run_label}");

148

1291

                    tracing::subscriber::with_default(subscriber, || {

149

1291

                        integration_test_inner(

150

1291

                            path,

151

1291

                            essence_base,

152

1291

                            extension,

153

1291

                            run_case,

154

1291

                            minion_discrete_threshold,

155

1291

                            conjure_solutions.clone(),

156

1291

                            accept,

157

158

1291

})

159

1291

                    .map_err(|err| std::io::Error::other(format!("{run_label}: {err}")))?;

160

161

162

163

164

165

848

    Ok(())

166

854

167

168

/// Runs an integration test for a given Conjure model by:

169

/// 1. Parsing the model from an Essence file.

170

/// 2. Rewriting the model according to predefined rule sets.

171

/// 3. Solving the model using the Minion solver and validating the solutions.

172

/// 4. Comparing generated rule traces with expected outputs.

173

///

174

/// This function operates in multiple stages:

175

///

176

/// - **Parsing Stage**

177

///   - **Stage 1a (Default)**: Reads the Essence model file and verifies that it parses correctly using `parser = "tree-sitter"`.

178

///   - **Stage 1b (Optional)**: Reads the Essence model file and verifies that it parses correctly using `parser = "via-conjure"`.

179

///

180

/// - **Rewrite Stage**

181

///   - **Stage 2a**: Applies a set of rules to the parsed model and validates the result.

182

///

183

/// - **Solution Stage**

184

///   - **Stage 3a (Default)**: Uses Minion to solve the model and save the solutions.

185

///   - **Stage 3b (ACCEPT only)**: Compares solutions against Conjure-generated solutions.

186

///

187

/// - **Rule Trace Validation Stage**

188

///   - **Stage 4a (Default)**: Checks that the generated rules match expected traces.

189

///

190

/// # Arguments

191

///

192

/// * `path` - The file path where the Essence model and other resources are located.

193

/// * `essence_base` - The base name of the Essence model file.

194

/// * `extension` - The file extension for the Essence model.

195

///

196

/// # Errors

197

///

198

/// Returns an error if any stage fails due to a mismatch with expected results or file I/O issues.

199

#[allow(clippy::unwrap_used)]

200

430

fn integration_test_inner(

201

430

    path: &str,

202

430

    essence_base: &str,

203

430

    extension: &str,

204

430

    run_case: RunCase<'_>,

205

430

    minion_discrete_threshold: usize,

206

430

    conjure_solutions: Option<Arc<Vec<BTreeMap<Name, Literal>>>>,

207

430

    accept: bool,

208

430

) -> Result<(), Box<dyn Error>> {

209

430

    let parser = run_case.parser;

210

430

    let rewriter = run_case.rewriter;

211

430

    let comprehension_expander = run_case.comprehension_expander;

212

430

    let solver_fam = run_case.solver;

213

430

    let case_name = run_case.case_name;

214

215

430

    let context: Arc<RwLock<Context<'static>>> = Default::default();

216

217

430

    set_current_parser(parser);

218

430

    set_current_rewriter(rewriter);

219

430

    set_comprehension_expander(comprehension_expander);

220

430

    set_current_solver_family(solver_fam);

221

430

    set_minion_discrete_threshold(minion_discrete_threshold);

222

223

    // File path

224

430

    let file_path = format!("{path}/{essence_base}.{extension}");

225

226

    // Stage 1a/1b: Parse the model using the selected parser.

227

430

    let parsed_model = match parser {

228

        Parser::TreeSitter => {

229

24

            let mut ctx = context.as_ref().write().unwrap();

230

24

            ctx.file_name = Some(format!("{path}/{essence_base}.{extension}"));

231

24

            parse_essence_file_native(&file_path, context.clone())?

232

233

406

        Parser::ViaConjure => parse_essence_file(&file_path, context.clone())?,

234

};

235

236

    // Stage 2a: Rewrite the model using the rule engine

237

430

    let mut extra_rules = vec![];

238

239

430

    if let SolverFamily::Sat(sat_encoding) = solver_fam {

240

99

        extra_rules.push(sat_encoding.as_rule_set());

241

331

242

243

430

    let mut rules_to_load = DEFAULT_RULE_SETS.to_vec();

244

430

    rules_to_load.extend(extra_rules);

245

246

430

    let rule_sets = resolve_rule_sets(solver_fam, &rules_to_load)?;

247

248

430

    let mut model = parsed_model;

249

250

430

    let rewritten_model = match rewriter {

251

430

        Rewriter::Naive => rewrite_naive(&model, &rule_sets, false)?,

252

        Rewriter::Morph => {

253

            let submodel = &mut model;

254

            let rules_grouped = get_rules_grouped(&rule_sets)

255

                .unwrap_or_else(|_| bug!("get_rule_priorities() failed!"))

256

                .into_iter()

257

                .map(|(_, rule)| rule.into_iter().map(|f| f.rule).collect_vec())

258

                .collect_vec();

259

260

            let engine = EngineBuilder::new()

261

                .set_selector(select_panic)

262

                .append_rule_groups(rules_grouped)

263

                .build();

264

            let (expr, symbol_table) =

265

                engine.morph(submodel.root().clone(), submodel.symbols().clone());

266

267

            *submodel.symbols_mut() = symbol_table;

268

            submodel.replace_root(expr);

269

            model.clone()

270

271

};

272

430

    let solver_input_file = None;

273

430

    let solver = match solver_fam {

274

286

        SolverFamily::Minion => Solver::new(Minion::default()),

275

99

        SolverFamily::Sat(_) => Solver::new(Sat::default()),

276

        #[cfg(feature = "smt")]

277

45

        SolverFamily::Smt(_) => Solver::new(Smt::default()),

278

};

279

280

430

    let solutions = {

281

430

        let solved = get_solutions(solver, rewritten_model, 0, &solver_input_file)?;

282

430

        save_solutions_json(&solved, path, case_name, solver_fam)?;

283

430

        solved

284

};

285

286

    // Stage 3b: Check solutions against Conjure when ACCEPT=true and validation is enabled.

287

430

    if accept && conjure_solutions.is_some() {

288

        let conjure_solutions = conjure_solutions

289

            .as_deref()

290

            .expect("conjure solutions should be present when Conjure validation is enabled");

291

292

        let username_solutions = normalize_solutions_for_comparison(&solutions);

293

        let conjure_solutions = normalize_solutions_for_comparison(conjure_solutions);

294

295

        let mut conjure_solutions_json = solutions_to_json(&conjure_solutions);

296

        let mut username_solutions_json = solutions_to_json(&username_solutions);

297

298

        conjure_solutions_json.sort_all_objects();

299

        username_solutions_json.sort_all_objects();

300

301

        assert_eq!(

302

            username_solutions_json, conjure_solutions_json,

303

            "Solutions (<) do not match conjure (>)!"

304

);

305

430

306

307

    // When ACCEPT=true, copy all generated files to expected

308

430

    if accept {

309

        // Always overwrite these ones. Unlike the rest, we don't need to selectively do these

310

        // based on the test results, so they don't get done later.

311

        copy_generated_to_expected(path, case_name, "solutions", "json", solver_fam)?;

312

        copy_human_trace_generated_to_expected(path, case_name, solver_fam)?;

313

430

314

315

    // Check Stage 3a (solutions)

316

430

    let expected_solutions_json = read_solutions_json(path, case_name, "expected", solver_fam)?;

317

430

    let username_solutions_json = solutions_to_json(&solutions);

318

430

    assert_eq!(username_solutions_json, expected_solutions_json);

319

320

    // TODO: Implement rule trace validation for morph

321

430

    match rewriter {

322

        Rewriter::Morph => {}

323

        Rewriter::Naive => {

324

430

            let generated = read_human_rule_trace(path, case_name, "generated", &solver_fam)?;

325

430

            let expected = read_human_rule_trace(path, case_name, "expected", &solver_fam)?;

326

327

430

            assert_eq!(

328

                expected, generated,

329

                "Generated rule trace does not match the expected trace!"

330

);

331

332

333

334

430

    save_stats_json(context, path, case_name, solver_fam)?;

335

336

430

    Ok(())

337

430

338

339

1291

fn run_case_name(

340

1291

    parser: Parser,

341

1291

    rewriter: Rewriter,

342

1291

    comprehension_expander: QuantifiedExpander,

343

1291

) -> String {

344

1291

    format!("{parser}-{rewriter}-{comprehension_expander}")

345

1291

346

347

fn clean_test_dir_for_accept(

348

    path: &str,

349

    essence_base: &str,

350

    extension: &str,

351

) -> Result<(), std::io::Error> {

352

    let input_filename = format!("{essence_base}.{extension}");

353

354

    for entry in std::fs::read_dir(path)? {

355

        let entry = entry?;

356

        let file_name = entry.file_name();

357

        let file_name = file_name.to_string_lossy();

358

        let entry_path = entry.path();

359

360

        if file_name == input_filename || file_name == "config.toml" {

361

            continue;

362

363

364

        if entry_path.is_dir() {

365

            std::fs::remove_dir_all(entry_path)?;

366

        } else {

367

            std::fs::remove_file(entry_path)?;

368

369

370

371

    Ok(())

372

373

374

fn copy_human_trace_generated_to_expected(

375

    path: &str,

376

    test_name: &str,

377

    solver: SolverFamily,

378

) -> Result<(), std::io::Error> {

379

    let solver_name = solver.as_str();

380

381

    std::fs::copy(

382

        format!("{path}/{test_name}-{solver_name}-generated-rule-trace.txt"),

383

        format!("{path}/{test_name}-{solver_name}-expected-rule-trace.txt"),

384

)?;

385

    Ok(())

386

387

388

fn copy_generated_to_expected(

389

    path: &str,

390

    test_name: &str,

391

    stage: &str,

392

    extension: &str,

393

    solver: SolverFamily,

394

) -> Result<(), std::io::Error> {

395

    let marker = solver.as_str();

396

397

    std::fs::copy(

398

        format!("{path}/{test_name}-{marker}.generated-{stage}.{extension}"),

399

        format!("{path}/{test_name}-{marker}.expected-{stage}.{extension}"),

400

)?;

401

    Ok(())

402

403

404

#[test]

405

3

fn assert_conjure_present() {

406

3

    conjure_cp_cli::find_conjure::conjure_executable().unwrap();

407

3

408

409

include!(concat!(env!("OUT_DIR"), "/gen_tests.rs"));