Grcov report - integration

1

#![allow(clippy::expect_used)]

2

use conjure_cp::bug;

3

use conjure_cp::rule_engine::get_rules_grouped;

4

use git_version as _;

5

6

use conjure_cp::defaults::DEFAULT_RULE_SETS;

7

use conjure_cp::parse::tree_sitter::parse_essence_file_native;

8

use conjure_cp::rule_engine::rewrite_naive;

9

use conjure_cp::solver::Solver;

10

use conjure_cp::solver::adaptors::*;

11

use conjure_cp_cli::utils::testing::{normalize_solutions_for_comparison, read_human_rule_trace};

12

use itertools::Itertools;

13

use std::collections::BTreeMap;

14

use std::env;

15

use std::error::Error;

16

use std::fs;

17

use std::fs::File;

18

use tracing_subscriber::{Layer, filter::EnvFilter, filter::FilterFn, fmt, layer::SubscriberExt};

19

use tree_morph::{helpers::select_panic, prelude::*};

20

21

use std::sync::Arc;

22

use std::sync::RwLock;

23

24

use conjure_cp::ast::{Literal, Name};

25

use conjure_cp::context::Context;

26

use conjure_cp::parse::tree_sitter::parse_essence_file;

27

use conjure_cp::rule_engine::resolve_rule_sets;

28

use conjure_cp::settings::{

29

    Parser, QuantifiedExpander, Rewriter, SolverFamily, set_comprehension_expander,

30

    set_current_parser, set_current_rewriter, set_current_solver_family,

31

    set_minion_discrete_threshold,

32

};

33

use conjure_cp_cli::utils::conjure::solutions_to_json;

34

use conjure_cp_cli::utils::conjure::{get_solutions, get_solutions_from_conjure};

35

use conjure_cp_cli::utils::testing::save_stats_json;

36

use conjure_cp_cli::utils::testing::{read_solutions_json, save_solutions_json};

37

#[allow(clippy::single_component_path_imports, unused_imports)]

38

use conjure_cp_rules;

39

use pretty_assertions::assert_eq;

40

use tests_integration::TestConfig;

41

42

#[derive(Clone, Copy, Debug)]

43

struct RunCase<'a> {

44

    parser: Parser,

45

    rewriter: Rewriter,

46

    comprehension_expander: QuantifiedExpander,

47

    solver: SolverFamily,

48

    case_name: &'a str,

49

50

51

1379

fn run_case_label(

52

1379

    path: &str,

53

1379

    essence_base: &str,

54

1379

    extension: &str,

55

1379

    run_case: RunCase<'_>,

56

1379

) -> String {

57

1379

    format!(

58

        "test_dir={path}, model={essence_base}.{extension}, parser={}, rewriter={}, comprehension_expander={}, solver={}",

59

        run_case.parser,

60

        run_case.rewriter,

61

        run_case.comprehension_expander,

62

1379

        run_case.solver.as_str()

63

64

1379

65

66

577

fn integration_test(path: &str, essence_base: &str, extension: &str) -> Result<(), Box<dyn Error>> {

67

577

    let accept = env::var("ACCEPT").unwrap_or("false".to_string()) == "true";

68

69

577

    if accept {

70

        clean_test_dir_for_accept(path, essence_base, extension)?;

71

577

72

73

577

    let file_config: TestConfig =

74

577

        if let Ok(config_contents) = fs::read_to_string(format!("{path}/config.toml")) {

75

577

            toml::from_str(&config_contents).unwrap()

76

        } else {

77

            Default::default()

78

};

79

80

577

    let config = file_config;

81

82

577

    let validate_with_conjure = config.validate_with_conjure;

83

577

    let minion_discrete_threshold = config.minion_discrete_threshold;

84

85

577

    let parsers = config

86

577

        .configured_parsers()

87

577

        .map_err(|err| std::io::Error::new(std::io::ErrorKind::InvalidInput, err))?;

88

577

    let rewriters = config

89

577

        .configured_rewriters()

90

577

        .map_err(|err| std::io::Error::new(std::io::ErrorKind::InvalidInput, err))?;

91

577

    let comprehension_expanders = config

92

577

        .configured_comprehension_expanders()

93

577

        .map_err(|err| std::io::Error::new(std::io::ErrorKind::InvalidInput, err))?;

94

577

    let solvers = config

95

577

        .configured_solvers()

96

577

        .map_err(|err| std::io::Error::new(std::io::ErrorKind::InvalidInput, err))?;

97

    // Conjure output depends only on the input model, so cache it once per test case.

98

577

    let model_path = format!("{path}/{essence_base}.{extension}");

99

577

    let conjure_solutions = if accept && validate_with_conjure {

100

        eprintln!("[integration] loading Conjure reference solutions for {model_path}");

101

        Some(Arc::new(

102

            get_solutions_from_conjure(&model_path, Default::default()).map_err(|err| {

103

                std::io::Error::other(format!(

104

                    "failed to fetch Conjure reference solutions for {model_path}: {err}"

105

))

106

            })?,

107

))

108

    } else {

109

577

        if accept && !validate_with_conjure {

110

            eprintln!("[integration] skipping Conjure validation for {model_path}");

111

577

112

577

        None

113

};

114

115

623

    for parser in parsers {

116

623

        for rewriter in rewriters.clone() {

117

679

            for comprehension_expander in comprehension_expanders.clone() {

118

921

                for solver in solvers.clone() {

119

921

                    let case_name = run_case_name(parser, rewriter, comprehension_expander);

120

921

                    let run_case = RunCase {

121

921

                        parser,

122

921

                        rewriter,

123

921

                        comprehension_expander,

124

921

                        solver,

125

921

                        case_name: case_name.as_str(),

126

921

};

127

921

                    let run_label = run_case_label(path, essence_base, extension, run_case);

128

463

                    eprintln!("[integration] running {run_label}");

129

463

                    integration_test_inner(

130

921

                        path,

131

463

                        essence_base,

132

921

                        extension,

133

921

                        run_case,

134

921

                        minion_discrete_threshold,

135

921

                        conjure_solutions.clone(),

136

921

                        accept,

137

458

138

921

                    .map_err(|err| std::io::Error::other(format!("{run_label}: {err}")))?;

139

458

140

52250

141

52250

142

52250

143

144

291

    Ok(())

145

749

146

458

147

/// Runs an integration test for a given Conjure model by:

148

/// 1. Parsing the model from an Essence file.

149

/// 2. Rewriting the model according to predefined rule sets.

150

/// 3. Solving the model using the Minion solver and validating the solutions.

151

/// 4. Comparing generated rule traces with expected outputs.

152

///

153

/// This function operates in multiple stages:

154

///

155

/// - **Parsing Stage**

156

///   - **Stage 1a (Default)**: Reads the Essence model file and verifies that it parses correctly using `parser = "tree-sitter"`.

157

///   - **Stage 1b (Optional)**: Reads the Essence model file and verifies that it parses correctly using `parser = "via-conjure"`.

158

///

159

/// - **Rewrite Stage**

160

///   - **Stage 2a**: Applies a set of rules to the parsed model and validates the result.

161

///

162

/// - **Solution Stage**

163

///   - **Stage 3a (Default)**: Uses Minion to solve the model and save the solutions.

164

///   - **Stage 3b (ACCEPT only)**: Compares solutions against Conjure-generated solutions.

165

///

166

/// - **Rule Trace Validation Stage**

167

///   - **Stage 4a (Default)**: Checks that the generated rules match expected traces.

168

///

169

/// # Arguments

170

///

171

/// * `path` - The file path where the Essence model and other resources are located.

172

/// * `essence_base` - The base name of the Essence model file.

173

/// * `extension` - The file extension for the Essence model.

174

///

175

/// # Errors

176

///

177

/// Returns an error if any stage fails due to a mismatch with expected results or file I/O issues.

178

#[allow(clippy::unwrap_used)]

179

463

fn integration_test_inner(

180

463

    path: &str,

181

463

    essence_base: &str,

182

463

    extension: &str,

183

463

    run_case: RunCase<'_>,

184

463

    minion_discrete_threshold: usize,

185

463

    conjure_solutions: Option<Arc<Vec<BTreeMap<Name, Literal>>>>,

186

463

    accept: bool,

187

463

) -> Result<(), Box<dyn Error>> {

188

463

    let parser = run_case.parser;

189

463

    let rewriter = run_case.rewriter;

190

463

    let comprehension_expander = run_case.comprehension_expander;

191

463

    let solver_fam = run_case.solver;

192

463

    let case_name = run_case.case_name;

193

194

463

    let context: Arc<RwLock<Context<'static>>> = Default::default();

195

196

463

    set_current_parser(parser);

197

463

    set_current_rewriter(rewriter);

198

463

    set_comprehension_expander(comprehension_expander);

199

463

    set_current_solver_family(solver_fam);

200

921

    set_minion_discrete_threshold(minion_discrete_threshold);

201

458

202

    // File path

203

921

    let file_path = format!("{path}/{essence_base}.{extension}");

204

458

205

    // Stage 1a/1b: Parse the model using the selected parser.

206

921

    let parsed_model = match parser {

207

458

        Parser::TreeSitter => {

208

491

            let mut ctx = context.as_ref().write().unwrap();

209

491

            ctx.file_name = Some(format!("{path}/{essence_base}.{extension}"));

210

491

            parse_essence_file_native(&file_path, context.clone())?

211

458

212

888

        Parser::ViaConjure => parse_essence_file(&file_path, context.clone())?,

213

458

};

214

463

    let should_validate_rule_trace = parsed_model.dominance.is_none();

215

921

    let generated_trace_path = format!(

216

        "{path}/{}-{}-generated-rule-trace.txt",

217

458

        case_name,

218

921

        solver_fam.as_str()

219

458

);

220

921

    if !should_validate_rule_trace {

221

463

        let _ = fs::remove_file(&generated_trace_path);

222

458

223

224

    // Stage 2a: Rewrite the model using the rule engine

225

463

    let mut extra_rules = vec![];

226

227

921

    if let SolverFamily::Sat(sat_encoding) = solver_fam {

228

118

        extra_rules.push(sat_encoding.as_rule_set());

229

373

230

28

231

491

    let mut rules_to_load = DEFAULT_RULE_SETS.to_vec();

232

463

    rules_to_load.extend(extra_rules);

233

430

234

463

    let rule_sets = resolve_rule_sets(solver_fam, &rules_to_load)?;

235

236

463

    let mut model = parsed_model;

237

458

238

463

    let rewritten_model = match rewriter {

239

916

        Rewriter::Naive if should_validate_rule_trace => {

240

571

            let file = File::create(&generated_trace_path)?;

241

803

            let subscriber = Arc::new(

242

458

                tracing_subscriber::registry().with(

243

916

                    fmt::layer()

244

916

                        .with_writer(file)

245

458

                        .with_level(false)

246

916

                        .without_time()

247

458

                        .with_target(false)

248

916

                        .with_filter(EnvFilter::new("rule_engine_human=trace"))

249

46298

                        .with_filter(FilterFn::new(|meta| meta.target() == "rule_engine_human")),

250

458

),

251

916

            ) as Arc<dyn tracing::Subscriber + Send + Sync>;

252

253

458

            tracing::subscriber::with_default(subscriber, || {

254

458

                rewrite_naive(&model, &rule_sets, false)

255

458

})?

256

257

5

        Rewriter::Naive => rewrite_naive(&model, &rule_sets, false)?,

258

        Rewriter::Morph => {

259

            let submodel = &mut model;

260

            let rules_grouped = get_rules_grouped(&rule_sets)

261

                .unwrap_or_else(|_| bug!("get_rule_priorities() failed!"))

262

                .into_iter()

263

                .map(|(_, rule)| rule.into_iter().map(|f| f.rule).collect_vec())

264

                .collect_vec();

265

266

            let engine = EngineBuilder::new()

267

                .set_selector(select_panic)

268

                .append_rule_groups(rules_grouped)

269

                .build();

270

            let (expr, symbol_table) =

271

                engine.morph(submodel.root().clone(), submodel.symbols().clone());

272

458

273

458

            *submodel.symbols_mut() = symbol_table;

274

292

            submodel.replace_root(expr);

275

113

            model.clone()

276

277

53

};

278

463

    let solver_input_file = None;

279

463

    let solver = match solver_fam {

280

750

        SolverFamily::Minion => Solver::new(Minion::default()),

281

576

        SolverFamily::Sat(_) => Solver::new(Sat::default()),

282

458

283

511

        SolverFamily::Smt(_) => Solver::new(Smt::default()),

284

};

285

286

463

    let solutions = {

287

921

        let solved = get_solutions(solver, rewritten_model, 0, &solver_input_file)?;

288

463

        save_solutions_json(&solved, path, case_name, solver_fam)?;

289

463

        solved

290

};

291

292

    // Stage 3b: Check solutions against Conjure when ACCEPT=true and validation is enabled.

293

463

    if accept && conjure_solutions.is_some() {

294

        let conjure_solutions = conjure_solutions

295

            .as_deref()

296

            .expect("conjure solutions should be present when Conjure validation is enabled");

297

298

        let username_solutions = normalize_solutions_for_comparison(&solutions);

299

        let conjure_solutions = normalize_solutions_for_comparison(conjure_solutions);

300

301

        let mut conjure_solutions_json = solutions_to_json(&conjure_solutions);

302

        let mut username_solutions_json = solutions_to_json(&username_solutions);

303

304

        conjure_solutions_json.sort_all_objects();

305

458

        username_solutions_json.sort_all_objects();

306

307

        assert_eq!(

308

458

            username_solutions_json, conjure_solutions_json,

309

            "Solutions (<) do not match conjure (>)!"

310

);

311

463

312

313

    // When ACCEPT=true, copy all generated files to expected

314

463

    if accept {

315

        // Always overwrite these ones. Unlike the rest, we don't need to selectively do these

316

        // based on the test results, so they don't get done later.

317

458

        copy_generated_to_expected(path, case_name, "solutions", "json", solver_fam)?;

318

458

        if should_validate_rule_trace {

319

            copy_human_trace_generated_to_expected(path, case_name, solver_fam)?;

320

321

921

322

323

    // Check Stage 3a (solutions)

324

921

    let expected_solutions_json = read_solutions_json(path, case_name, "expected", solver_fam)?;

325

921

    let username_solutions_json = solutions_to_json(&solutions);

326

463

    assert_eq!(username_solutions_json, expected_solutions_json);

327

458

328

    // TODO: Implement rule trace validation for morph

329

463

    match rewriter {

330

        Rewriter::Morph => {}

331

        Rewriter::Naive => {

332

463

            if should_validate_rule_trace {

333

458

                let generated = read_human_rule_trace(path, case_name, "generated", &solver_fam)?;

334

916

                let expected = read_human_rule_trace(path, case_name, "expected", &solver_fam)?;

335

336

916

                assert_eq!(

337

458

                    expected, generated,

338

                    "Generated rule trace does not match the expected trace!"

339

916

);

340

921

341

916

342

916

343

916

344

1379

    save_stats_json(context, path, case_name, solver_fam)?;

345

916

346

463

    Ok(())

347

463

348

349

463

fn run_case_name(

350

463

    parser: Parser,

351

463

    rewriter: Rewriter,

352

463

    comprehension_expander: QuantifiedExpander,

353

463

) -> String {

354

463

    format!("{parser}-{rewriter}-{comprehension_expander}")

355

463

356

357

fn clean_test_dir_for_accept(

358

    path: &str,

359

    essence_base: &str,

360

    extension: &str,

361

) -> Result<(), std::io::Error> {

362

    let input_filename = format!("{essence_base}.{extension}");

363

364

    for entry in std::fs::read_dir(path)? {

365

        let entry = entry?;

366

        let file_name = entry.file_name();

367

        let file_name = file_name.to_string_lossy();

368

        let entry_path = entry.path();

369

370

        if file_name == input_filename || file_name == "config.toml" {

371

            continue;

372

373

374

        if entry_path.is_dir() {

375

            std::fs::remove_dir_all(entry_path)?;

376

        } else {

377

            std::fs::remove_file(entry_path)?;

378

379

380

381

    Ok(())

382

383

384

fn copy_human_trace_generated_to_expected(

385

    path: &str,

386

    test_name: &str,

387

    solver: SolverFamily,

388

) -> Result<(), std::io::Error> {

389

    let solver_name = solver.as_str();

390

391

    std::fs::copy(

392

        format!("{path}/{test_name}-{solver_name}-generated-rule-trace.txt"),

393

        format!("{path}/{test_name}-{solver_name}-expected-rule-trace.txt"),

394

)?;

395

    Ok(())

396

397

398

fn copy_generated_to_expected(

399

    path: &str,

400

    test_name: &str,

401

    stage: &str,

402

    extension: &str,

403

    solver: SolverFamily,

404

) -> Result<(), std::io::Error> {

405

2

    let marker = solver.as_str();

406

2

407

2

    std::fs::copy(

408

        format!("{path}/{test_name}-{marker}.generated-{stage}.{extension}"),

409

        format!("{path}/{test_name}-{marker}.expected-{stage}.{extension}"),

410

)?;

411

    Ok(())

412

413

414

#[test]

415

1

fn assert_conjure_present() {

416

1

    conjure_cp_cli::find_conjure::conjure_executable().unwrap();

417

1

418

419

include!(concat!(env!("OUT_DIR"), "/gen_tests.rs"));