Grcov report - test

1

#![allow(unused)]

2

3

use conjure_cp::settings::{

4

    Channelling, Heuristic, Parser, QuantifiedExpander, Rewriter, SolverFamily,

5

};

6

use serde::de::{self, Visitor};

7

use serde::{Deserialize, Serialize};

8

use std::collections::BTreeMap;

9

use std::fmt;

10

use std::fs;

11

use std::io;

12

use std::path::Path;

13

use std::str::FromStr;

14

use std::time::Duration;

15

use toml_edit::{DocumentMut, Item, Table, value};

16

17

pub const STATS_FILE_NAME: &str = "stats.toml";

18

19

4960

pub fn stats_path(test_dir: &Path) -> std::path::PathBuf {

20

4960

    test_dir.join(STATS_FILE_NAME)

21

4960

22

23

/// Starts a fresh stats snapshot for an exhaustive run.

24

2

pub fn reset_stats_for_run(path: &Path) -> io::Result<()> {

25

2

    write_canonical_stats(path, &TestRunStats::default())

26

2

27

28

fn read_toml_document_or_empty(path: &Path) -> io::Result<DocumentMut> {

29

    if path.exists() {

30

        let contents = fs::read_to_string(path)?;

31

        if contents.trim().is_empty() {

32

            Ok(DocumentMut::new())

33

        } else {

34

            contents

35

                .parse::<DocumentMut>()

36

                .map_err(|err| io::Error::new(io::ErrorKind::InvalidData, err))

37

38

    } else {

39

        Ok(DocumentMut::new())

40

41

42

43

fn write_toml_document(path: &Path, document: &DocumentMut) -> io::Result<()> {

44

    let mut contents = document.to_string();

45

    contents.truncate(contents.trim_end_matches('\n').len());

46

    contents.push('\n');

47

    fs::write(path, contents)

48

49

50

9

fn quoted_toml_string(value: &str) -> String {

51

9

    toml::Value::String(value.to_string()).to_string()

52

9

53

54

15

fn canonical_float(value: f64) -> String {

55

15

    let rounded = (value * 1_000_000.0).round() / 1_000_000.0;

56

15

    let mut rendered = format!("{rounded:.6}");

57

24

    while rendered.ends_with('0') {

58

9

        rendered.pop();

59

60

15

    if rendered.ends_with('.') {

61

        rendered.push('0');

62

15

63

15

    if rendered == "-0.0" {

64

        "0.0".to_string()

65

    } else {

66

15

        rendered

67

68

15

69

70

6

fn canonical_rule_key(rule: &str) -> String {

71

6

    if !rule.is_empty()

72

6

        && rule

73

6

            .chars()

74

54

            .all(|character| character.is_ascii_alphanumeric() || matches!(character, '_' | '-'))

75

76

6

        rule.to_string()

77

    } else {

78

        quoted_toml_string(rule)

79

80

6

81

82

// toml_edit's Index impl panics on missing keys, so use .get() before creating tables.

83

fn ensure_table(document: &mut DocumentMut, key: &str) {

84

    if document.get(key).is_some_and(|item| item.is_table()) {

85

        return;

86

87

    document[key] = Item::Table(Table::new());

88

89

90

fn ensure_nested_table(document: &mut DocumentMut, keys: &[&str]) {

91

    let (head, tail) = keys.split_first().expect("table path must not be empty");

92

    ensure_table(document, head);

93

    let mut table = document[head].as_table_mut().expect("table exists");

94

    for key in tail {

95

        if table.get(key).is_some_and(|item| item.is_table()) {

96

            table = table[key].as_table_mut().expect("table exists");

97

            continue;

98

99

        table[*key] = Item::Table(Table::new());

100

        table = table[key].as_table_mut().expect("table exists");

101

102

103

104

16881

fn parse_values<T>(values: &[String]) -> Result<Vec<T>, String>

105

16881

where

106

16881

    T: FromStr<Err = String>,

107

108

20079

    values.iter().map(|value| value.parse()).collect()

109

16881

110

111

14534

fn deserialize_string_or_vec<'de, D>(deserializer: D) -> Result<Vec<String>, D::Error>

112

14534

where

113

14534

    D: serde::Deserializer<'de>,

114

115

    #[derive(Deserialize)]

116

    #[serde(untagged)]

117

    enum StringOrVec {

118

        String(String),

119

        Vec(Vec<String>),

120

121

122

14534

    Ok(match StringOrVec::deserialize(deserializer)? {

123

12211

        StringOrVec::String(s) => vec![s],

124

2323

        StringOrVec::Vec(v) => v,

125

})

126

14534

127

128

4101

fn default_skip_conjure_validation() -> String {

129

4101

    String::new()

130

4101

131

132

4318

fn default_skip() -> String {

133

4318

    String::new()

134

4318

135

136

8999

fn default_minion_discrete_threshold() -> usize {

137

8999

    conjure_cp::settings::DEFAULT_MINION_DISCRETE_THRESHOLD

138

8999

139

140

3824

fn deserialise_expected_time<'de, D>(deserializer: D) -> Result<Option<u64>, D::Error>

141

3824

where

142

3824

    D: serde::Deserializer<'de>,

143

144

3824

    Option::<u64>::deserialize(deserializer)

145

3824

146

147

1155

fn default_number_of_solutions() -> NumberOfSolutions {

148

1155

    NumberOfSolutions::All

149

1155

150

151

4337

fn default_keep_intermediate_solutions() -> bool {

152

4337

    false

153

4337

154

155

2005

fn deserialise_number_of_solutions<'de, D>(deserializer: D) -> Result<NumberOfSolutions, D::Error>

156

2005

where

157

2005

    D: serde::Deserializer<'de>,

158

159

2005

    deserializer.deserialize_any(NumberOfSolutionsVisitor)

160

2005

161

162

struct NumberOfSolutionsVisitor;

163

164

impl<'de> Visitor<'de> for NumberOfSolutionsVisitor {

165

    type Value = NumberOfSolutions;

166

167

    fn expecting(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {

168

        formatter.write_str("a positive integer, the string \"all\", or the string \"skip\"")

169

170

171

79

    fn visit_u64<E>(self, value: u64) -> Result<Self::Value, E>

172

79

    where

173

79

        E: de::Error,

174

175

79

        let value = i32::try_from(value).map_err(|err| {

176

            E::custom(format!(

177

                "number-of-solutions is too large for the solver limit: {err}"

178

))

179

        })?;

180

181

79

        if value == 0 {

182

            return Err(E::custom(

183

                "number-of-solutions must be positive, or the string \"all\" or \"skip\"",

184

));

185

79

186

187

79

        Ok(NumberOfSolutions::Limit(value))

188

79

189

190

79

    fn visit_i64<E>(self, value: i64) -> Result<Self::Value, E>

191

79

    where

192

79

        E: de::Error,

193

194

79

        let value = u64::try_from(value).map_err(|_| {

195

            E::custom("number-of-solutions must be positive, or the string \"all\" or \"skip\"")

196

        })?;

197

198

79

        self.visit_u64(value)

199

79

200

201

1926

    fn visit_str<E>(self, value: &str) -> Result<Self::Value, E>

202

1926

    where

203

1926

        E: de::Error,

204

205

1926

        match value {

206

1910

            "all" => Ok(NumberOfSolutions::All),

207

16

            "skip" => Ok(NumberOfSolutions::Skip),

208

            _ => Err(E::custom(format!(

209

                "invalid number-of-solutions value '{value}', expected a positive integer, \"all\", or \"skip\""

210

            ))),

211

212

1926

213

214

215

/// Rounds an observed runtime into the coarse `expected-time` buckets used by test configs,

216

/// such as `1`, `5`, `10`, `30`, `60`, and so on.

217

pub fn round_expected_time(duration: Duration) -> u64 {

218

    let seconds = duration.as_secs_f64();

219

220

    if seconds <= 1.0 {

221

222

    } else if seconds <= 5.0 {

223

224

    } else if seconds <= 10.0 {

225

226

    } else {

227

        ((seconds / 30.0).ceil() as u64) * 30

228

229

230

231

/// Inserts or updates the `expected-time` entry in a test `stats.toml`.

232

1

pub fn upsert_expected_time_stats(path: &Path, expected_time: u64) -> io::Result<()> {

233

1

    update_canonical_stats(path, |stats| stats.expected_time = Some(expected_time))

234

1

235

236

/// Inserts or updates the `expected-time` entry in a test `config.toml`.

237

///

238

/// Custom tests still keep their expected-time metadata in `config.toml`; exhaustive tests use

239

/// `stats.toml`.

240

pub fn upsert_expected_time_config(path: &Path, expected_time: u64) -> io::Result<()> {

241

    let expected_time = i64::try_from(expected_time).map_err(|err| {

242

        io::Error::new(

243

            io::ErrorKind::InvalidInput,

244

            format!("expected-time is too large to write to TOML: {err}"),

245

246

    })?;

247

    let mut document = read_toml_document_or_empty(path)?;

248

    document["expected-time"] = value(expected_time);

249

    write_toml_document(path, &document)

250

251

252

/// Inserts or updates the latest observed exhaustive status in a test `stats.toml`.

253

1

pub fn upsert_status_stats(path: &Path, status: &str) -> io::Result<()> {

254

1

    update_canonical_stats(path, |stats| stats.status = Some(status.to_string()))

255

1

256

257

/// Inserts or updates the latest observed status for one part of an exhaustive test.

258

1

pub fn upsert_tool_status_stats(path: &Path, tool: &str, status: &str) -> io::Result<()> {

259

1

    if tool != "conjure" {

260

        return Err(io::Error::new(

261

            io::ErrorKind::InvalidInput,

262

            format!("'{tool}' does not have shared tool stats"),

263

));

264

1

265

1

    update_canonical_stats(path, |stats| {

266

1

        stats.conjure.status = Some(status.to_string());

267

1

})

268

1

269

270

/// Identifies one configured exhaustive-test run.

271

#[derive(Clone, Debug, Eq, PartialEq)]

272

pub struct RecordedRunConfig {

273

    pub parser: String,

274

    pub rewriter: String,

275

    pub comprehension_expander: String,

276

    pub heuristic: String,

277

    pub channelling: String,

278

    pub seed: u64,

279

    pub solver_seed: u32,

280

    pub solver: String,

281

282

283

/// Timing measurements recorded from one accepted Conjure reference run.

284

#[derive(Clone, Copy, Debug)]

285

pub struct RecordedConjureStats {

286

    /// Wall-clock time of the complete `conjure solve` command, in seconds.

287

    pub conjure_wall_clock_time: f64,

288

    /// Total Conjure plus Savile Row translation time, in seconds.

289

    pub conjure_translation_time: f64,

290

    /// Time spent by Conjure before Savile Row is invoked, in seconds.

291

    pub conjure_driver_translation_time: f64,

292

    /// Time spent by Savile Row during reference translation, in seconds.

293

    pub savilerow_translation_time: f64,

294

    /// Time spent solving the Conjure plus Savile Row reference model, in seconds.

295

    pub conjure_solve_time: f64,

296

297

298

/// Inserts or updates the recorded Conjure reference timings in a test `stats.toml`.

299

1

pub fn upsert_conjure_timing_stats(path: &Path, stats: RecordedConjureStats) -> io::Result<()> {

300

1

    update_canonical_stats(path, |current| {

301

1

        current.conjure.wall_clock_time = Some(stats.conjure_wall_clock_time);

302

1

        current.conjure.translation_time = Some(stats.conjure_translation_time);

303

1

        current.conjure.conjure_translation_time = Some(stats.conjure_driver_translation_time);

304

1

        current.conjure.savilerow_translation_time = Some(stats.savilerow_translation_time);

305

1

        current.conjure.solve_time = Some(stats.conjure_solve_time);

306

1

})

307

1

308

309

13

fn rule_trace_rules_by_count_desc(

310

13

    rules: &std::collections::BTreeMap<String, u64>,

311

13

) -> Vec<(&String, u64)> {

312

13

    let mut sorted_rules: Vec<_> = rules.iter().map(|(rule, count)| (rule, *count)).collect();

313

13

    sorted_rules.sort_by(|(name_a, count_a), (name_b, count_b)| {

314

        count_b.cmp(count_a).then_with(|| name_a.cmp(name_b))

315

});

316

13

    sorted_rules

317

13

318

319

/// Recorded exhaustive-run metadata for one test directory.

320

#[derive(Deserialize, Debug, Default)]

321

#[serde(default)]

322

#[serde(deny_unknown_fields)]

323

pub struct TestRunStats {

324

    /// Latest observed overall status, such as `ok`, `fail`, or `timeout(N)`.

325

    pub status: Option<String>,

326

327

    /// Coarse expected wall time bucket used by MAX_EXPECTED_TIME test selection.

328

    #[serde(

329

        default,

330

        rename = "expected-time",

331

        deserialize_with = "deserialise_expected_time"

332

)]

333

    pub expected_time: Option<u64>,

334

335

    /// Metadata recorded for the Conjure plus Savile Row reference run.

336

    pub conjure: RecordedToolStats,

337

338

    /// One measured run per compiled model, including each expanded `x` choice.

339

    pub runs: Vec<RecordedRunStats>,

340

341

342

/// Flat timing and resolved modelling choices for one actual exhaustive model.

343

#[derive(Deserialize, Serialize, Debug, Default)]

344

#[serde(default, deny_unknown_fields, rename_all = "kebab-case")]

345

pub struct RecordedRunStats {

346

    pub parser: String,

347

    pub rewriter: String,

348

    pub comprehension_expander: String,

349

    pub heuristic: String,

350

    pub channelling: String,

351

    pub seed: u64,

352

    pub solver_seed: u32,

353

    pub solver: String,

354

    pub case_name: String,

355

    pub model_index: usize,

356

    /// Full realised choice indices and labels, rather than the replay prefix.

357

    pub choice_path: Vec<usize>,

358

    pub choices: Vec<String>,

359

    /// Distinct representation rules present in the compiled model.

360

    pub representations: Vec<String>,

361

    pub minion_discrete_threshold: Option<usize>,

362

    pub sat_encoding_amo: Vec<String>,

363

    pub sat_encoding_cardinality: Vec<String>,

364

    pub sat_encoding_pb: Vec<String>,

365

    pub sat_encoding_table: Vec<String>,

366

    pub sat_encoding_element: Vec<String>,

367

    pub sat_encoding_alldifferent: Vec<String>,

368

    pub status: String,

369

    /// Zero means all solutions; absent means translation only.

370

    pub solution_limit: Option<i32>,

371

    /// Wall time for parsing, instantiation and rewriting.

372

    pub translation_time: Option<f64>,

373

    /// Wall time for complete solution collection and output.

374

    pub solve_time: Option<f64>,

375

    pub backend_time: Option<f64>,

376

    /// Solver-call wall time, including callbacks and solver-time rewrites.

377

    pub search_time: Option<f64>,

378

    pub solution_processing_time: Option<f64>,

379

    /// Rule counts for this compiled model; written as dotted keys by the canonical writer.

380

    #[serde(skip_serializing)]

381

    pub rule_trace: RecordedRuleTraceStats,

382

383

384

/// Recorded rule counts for one compiled model.

385

#[derive(Deserialize, Debug, Default)]

386

#[serde(default)]

387

#[serde(deny_unknown_fields)]

388

pub struct RecordedRuleTraceStats {

389

    #[serde(rename = "attempts")]

390

    pub total_rule_attempts: Option<u64>,

391

    #[serde(rename = "applications")]

392

    pub total_rule_applications: Option<u64>,

393

    pub rules: std::collections::BTreeMap<String, u64>,

394

395

396

1785

pub fn read_stats_or_default(path: &Path) -> io::Result<TestRunStats> {

397

1785

    if path.exists() {

398

1784

        let contents = fs::read_to_string(path)?;

399

1784

        toml::from_str(&contents).map_err(|err| io::Error::new(io::ErrorKind::InvalidData, err))

400

    } else {

401

1

        Ok(TestRunStats::default())

402

403

1785

404

405

/// Recorded status and timings for one implementation in a test config.

406

#[derive(Deserialize, Debug, Default)]

407

#[serde(default)]

408

#[serde(deny_unknown_fields)]

409

pub struct RecordedToolStats {

410

    /// Latest observed status, such as `ok`, `fail`, or `timeout(N)`.

411

    pub status: Option<String>,

412

413

    /// Translation time in seconds.

414

    #[serde(rename = "translation-time")]

415

    pub translation_time: Option<f64>,

416

417

    /// Solver time in seconds.

418

    #[serde(rename = "solve-time")]

419

    pub solve_time: Option<f64>,

420

421

    /// Wall-clock time of the complete tool command, in seconds, when available.

422

    #[serde(rename = "wall-clock-time")]

423

    pub wall_clock_time: Option<f64>,

424

425

    /// Conjure-only translation time in seconds, when available.

426

    #[serde(rename = "conjure-translation-time")]

427

    pub conjure_translation_time: Option<f64>,

428

429

    /// Savile Row translation time in seconds, when available.

430

    #[serde(rename = "savilerow-translation-time")]

431

    pub savilerow_translation_time: Option<f64>,

432

433

434

/// Replace this configuration's model rows, removing stale portfolio entries.

435

5

pub fn replace_config_run_stats(

436

5

    path: &Path,

437

5

    config: &RecordedRunConfig,

438

5

    rows: Vec<RecordedRunStats>,

439

5

) -> io::Result<()> {

440

5

    update_canonical_stats(path, |stats| {

441

7

        stats.runs.retain(|row| {

442

7

            !(row.parser == config.parser

443

7

                && row.rewriter == config.rewriter

444

7

                && row.comprehension_expander == config.comprehension_expander

445

7

                && row.heuristic == config.heuristic

446

7

                && row.channelling == config.channelling

447

7

                && row.seed == config.seed

448

7

                && row.solver_seed == config.solver_seed

449

4

                && row.solver == config.solver)

450

7

});

451

5

        stats.runs.extend(rows);

452

8

        stats.runs.sort_by(|a, b| {

453

8

454

8

                &a.parser,

455

8

                &a.rewriter,

456

8

                &a.comprehension_expander,

457

8

                &a.heuristic,

458

8

                &a.channelling,

459

8

                a.seed,

460

8

                a.solver_seed,

461

8

                &a.solver,

462

8

                a.model_index,

463

8

464

8

                .cmp(&(

465

8

                    &b.parser,

466

8

                    &b.rewriter,

467

8

                    &b.comprehension_expander,

468

8

                    &b.heuristic,

469

8

                    &b.channelling,

470

8

                    b.seed,

471

8

                    b.solver_seed,

472

8

                    &b.solver,

473

8

                    b.model_index,

474

8

))

475

8

});

476

5

})

477

5

478

479

9

fn update_canonical_stats(path: &Path, update: impl FnOnce(&mut TestRunStats)) -> io::Result<()> {

480

9

    let mut stats = read_stats_or_default(path)?;

481

9

    update(&mut stats);

482

9

    write_canonical_stats(path, &stats)

483

9

484

485

20

fn push_optional_float(contents: &mut String, key: &str, value: Option<f64>) {

486

20

    if let Some(value) = value {

487

15

        contents.push_str(key);

488

15

        contents.push_str(" = ");

489

15

        contents.push_str(&canonical_float(value));

490

15

        contents.push('\n');

491

5

492

20

493

494

11

fn write_canonical_stats(path: &Path, stats: &TestRunStats) -> io::Result<()> {

495

11

    let mut contents = String::new();

496

497

11

    if let Some(expected_time) = stats.expected_time {

498

6

        contents.push_str(&format!("expected-time = {expected_time}\n"));

499

5

500

11

    if let Some(status) = &stats.status {

501

5

        contents.push_str(&format!("status = {}\n", quoted_toml_string(status)));

502

6

503

504

11

    let conjure = &stats.conjure;

505

11

    let has_conjure_stats = conjure.status.is_some()

506

7

        || conjure.translation_time.is_some()

507

7

        || conjure.solve_time.is_some()

508

7

        || conjure.wall_clock_time.is_some()

509

7

        || conjure.conjure_translation_time.is_some()

510

7

        || conjure.savilerow_translation_time.is_some();

511

11

    if has_conjure_stats {

512

4

        if !contents.is_empty() {

513

4

            contents.push('\n');

514

515

4

        contents.push_str("[conjure]\n");

516

4

        if let Some(status) = &conjure.status {

517

4

            contents.push_str(&format!("status = {}\n", quoted_toml_string(status)));

518

519

4

        push_optional_float(&mut contents, "wall-clock-time", conjure.wall_clock_time);

520

4

        push_optional_float(&mut contents, "translation-time", conjure.translation_time);

521

4

        push_optional_float(

522

4

            &mut contents,

523

4

            "conjure-translation-time",

524

4

            conjure.conjure_translation_time,

525

4

);

526

4

        push_optional_float(

527

4

            &mut contents,

528

4

            "savilerow-translation-time",

529

4

            conjure.savilerow_translation_time,

530

4

);

531

4

        push_optional_float(&mut contents, "solve-time", conjure.solve_time);

532

7

533

534

11

    for run in &stats.runs {

535

13

        contents.push_str("\n[[runs]]\n");

536

13

        contents.push_str(

537

13

            &toml::to_string(run).map_err(|err| io::Error::new(io::ErrorKind::InvalidData, err))?,

538

);

539

540

13

        if run.rule_trace.total_rule_attempts.is_some()

541

7

            || run.rule_trace.total_rule_applications.is_some()

542

7

            || !run.rule_trace.rules.is_empty()

543

544

6

            contents.push('\n');

545

7

546

13

        if let Some(attempts) = run.rule_trace.total_rule_attempts {

547

6

            contents.push_str(&format!("rule-trace.attempts = {attempts}\n"));

548

7

549

13

        if let Some(applications) = run.rule_trace.total_rule_applications {

550

6

            contents.push_str(&format!("rule-trace.applications = {applications}\n"));

551

7

552

13

        for (rule, count) in rule_trace_rules_by_count_desc(&run.rule_trace.rules) {

553

6

            contents.push_str(&format!(

554

6

                "rule-trace.rules.{} = {count}\n",

555

6

                canonical_rule_key(rule)

556

6

));

557

558

559

560

11

    fs::write(path, contents)

561

11

562

563

/// Solution search limit requested by an exhaustive test.

564

#[derive(Clone, Copy, Debug, Eq, PartialEq)]

565

pub enum NumberOfSolutions {

566

    /// Search for every solution.

567

    All,

568

    /// Stop after the given number of solutions.

569

    Limit(i32),

570

    /// Do not run the solver for this exhaustive test.

571

    Skip,

572

573

574

impl NumberOfSolutions {

575

    /// Converts the config value into the solver API limit, where `0` means all solutions.

576

377820

    pub fn as_solver_limit(self) -> Option<i32> {

577

377820

        match self {

578

369753

            Self::All => Some(0),

579

7992

            Self::Limit(limit) => Some(limit),

580

75

            Self::Skip => None,

581

582

377820

583

584

585

/// Per-backend modelling and trace choices for an exhaustive fixture.

586

#[derive(Deserialize, Debug, Default)]

587

#[serde(default, deny_unknown_fields)]

588

pub struct SolverOptions {

589

    /// Override the fixture's modelling heuristic for this backend.

590

    #[serde(deserialize_with = "deserialize_string_or_vec")]

591

    pub heuristic: Vec<String>,

592

    /// Override the fixture's representation-sharing policy for this backend.

593

    #[serde(deserialize_with = "deserialize_string_or_vec")]

594

    pub channelling: Vec<String>,

595

    /// Override the fixture's trace level for this backend.

596

    #[serde(rename = "trace-level")]

597

    pub trace_level: Option<TraceLevel>,

598

599

600

#[derive(Deserialize, Debug)]

601

#[serde(default)]

602

#[serde(deny_unknown_fields)]

603

pub struct TestConfig {

604

    #[serde(

605

        default,

606

        rename = "parser",

607

        deserialize_with = "deserialize_string_or_vec"

608

)]

609

    pub parser: Vec<String>, // Stage 1a: list of parsers (tree-sitter or via-conjure)

610

611

    #[serde(

612

        default,

613

        rename = "rewriter",

614

        deserialize_with = "deserialize_string_or_vec"

615

)]

616

    pub rewriter: Vec<String>,

617

    #[serde(

618

        default,

619

        rename = "comprehension-expander",

620

        deserialize_with = "deserialize_string_or_vec"

621

)]

622

    pub comprehension_expander: Vec<String>,

623

    #[serde(

624

        default,

625

        rename = "heuristic",

626

        deserialize_with = "deserialize_string_or_vec"

627

)]

628

    pub heuristic: Vec<String>,

629

    #[serde(

630

        default,

631

        rename = "channelling",

632

        deserialize_with = "deserialize_string_or_vec"

633

)]

634

    pub channelling: Vec<String>,

635

    #[serde(default, rename = "seed")]

636

    pub seed: u64,

637

    #[serde(default, rename = "solver-seed")]

638

    pub solver_seed: u32,

639

    #[serde(

640

        default,

641

        rename = "solver",

642

        deserialize_with = "deserialize_string_or_vec"

643

)]

644

    pub solver: Vec<String>,

645

646

    /// Backend-specific modelling and trace overrides.

647

    #[serde(default, rename = "solver-options")]

648

    pub solver_options: BTreeMap<String, SolverOptions>,

649

650

    #[serde(

651

        default,

652

        rename = "extra-rule-sets",

653

        deserialize_with = "deserialize_string_or_vec"

654

)]

655

    pub extra_rule_sets: Vec<String>,

656

657

    #[serde(

658

        default = "default_minion_discrete_threshold",

659

        rename = "minion-discrete-threshold"

660

)]

661

    pub minion_discrete_threshold: usize,

662

663

    #[serde(

664

        default = "default_skip_conjure_validation",

665

        rename = "skip-conjure-validation"

666

)]

667

    pub skip_conjure_validation: String,

668

669

    #[serde(

670

        default = "default_number_of_solutions",

671

        rename = "number-of-solutions",

672

        deserialize_with = "deserialise_number_of_solutions"

673

)]

674

    pub number_of_solutions: NumberOfSolutions,

675

676

    #[serde(

677

        default = "default_keep_intermediate_solutions",

678

        rename = "keep-intermediate-solutions"

679

)]

680

    pub keep_intermediate_solutions: bool,

681

682

    /// How much of the rule trace each run records. See [`TraceLevel`].

683

    #[serde(default, rename = "trace-level")]

684

    pub trace_level: TraceLevel,

685

686

    /// Empty `skip` runs the test; a non-empty string ignores it and records why.

687

    #[serde(default = "default_skip")]

688

    pub skip: String,

689

690

    #[serde(

691

        default,

692

        rename = "expected-time",

693

        deserialize_with = "deserialise_expected_time"

694

)]

695

    pub expected_time: Option<u64>,

696

697

698

/// The rule trace recorded by each exhaustive run.

699

#[derive(Deserialize, Debug, Default, Clone, Copy, PartialEq, Eq)]

700

#[serde(rename_all = "lowercase")]

701

pub enum TraceLevel {

702

    /// Disable rule trace snapshots and aggregate counts.

703

    None,

704

    /// Write the human-readable trace of every rule application and compare it with the

705

    /// expected snapshot.

706

    #[default]

707

    Full,

708

    /// Count rule applications in memory and record only the totals in `stats.toml`. Much

709

    /// cheaper for large fixtures, at the cost of the trace snapshot.

710

    Aggregate,

711

712

713

impl Default for TestConfig {

714

4657

    fn default() -> Self {

715

4657

        Self {

716

4657

            skip: String::new(),

717

4657

            parser: vec!["tree-sitter".to_string(), "via-conjure".to_string()],

718

4657

            rewriter: vec!["optimised".to_string()],

719

4657

            comprehension_expander: vec!["auto".to_string()],

720

4657

            heuristic: vec!["x".to_string()],

721

4657

            channelling: vec!["no".to_string()],

722

4657

            seed: 0,

723

4657

            solver_seed: 0,

724

            // How a model is expressed for a solver -- which SAT integer encoding, which Z3

725

            // integer theory -- is a representation choice, so the `x` heuristic above enumerates

726

            // those combinations rather than them being separate solvers here.

727

4657

            solver: vec!["minion".to_string(), "sat".to_string(), "z3".to_string()],

728

4657

            solver_options: BTreeMap::new(),

729

4657

            extra_rule_sets: Vec::new(),

730

4657

            minion_discrete_threshold: default_minion_discrete_threshold(),

731

4657

            skip_conjure_validation: String::new(),

732

4657

            number_of_solutions: NumberOfSolutions::All,

733

4657

            keep_intermediate_solutions: false,

734

4657

            trace_level: TraceLevel::Full,

735

4657

            expected_time: None,

736

4657

737

4657

738

739

740

impl TestConfig {

741

    /// Complete searches share one golden across all configurations.

742

1773

    pub fn uses_shared_solution_snapshot(&self) -> bool {

743

1773

        matches!(self.number_of_solutions, NumberOfSolutions::All)

744

1773

745

746

    /// Empty `skip-conjure-validation` runs Conjure reference validation during accept.

747

1773

    pub fn should_skip_conjure_validation(&self) -> bool {

748

1773

        !self.skip_conjure_validation.is_empty()

749

1773

750

751

    /// Empty `skip` runs the test; a non-empty string ignores it.

752

    pub fn should_skip(&self) -> bool {

753

        !self.skip.is_empty()

754

755

756

2014

    pub fn skip_reason(&self) -> Option<&str> {

757

2014

        if self.skip.is_empty() {

758

1990

            None

759

        } else {

760

24

            Some(self.skip.as_str())

761

762

2014

763

764

2262

    pub fn configured_parsers(&self) -> Result<Vec<Parser>, String> {

765

2262

        parse_values(&self.parser)

766

2262

767

768

1773

    pub fn configured_rewriters(&self) -> Result<Vec<Rewriter>, String> {

769

1773

        if self.rewriter.is_empty() {

770

            return Err("setting 'rewriter' has no values".to_string());

771

1773

772

773

1773

        parse_values(&self.rewriter)

774

1773

775

776

1773

    pub fn configured_comprehension_expanders(&self) -> Result<Vec<QuantifiedExpander>, String> {

777

1773

        let values = if self.comprehension_expander.is_empty() {

778

            vec!["auto".to_string()]

779

        } else {

780

1773

            self.comprehension_expander.clone()

781

};

782

783

1773

        parse_values(&values)

784

1773

785

786

2898

    pub fn configured_heuristics(&self) -> Result<Vec<Heuristic>, String> {

787

2898

        let values = if self.heuristic.is_empty() {

788

1035

            vec!["x".to_string()]

789

        } else {

790

1863

            self.heuristic.clone()

791

};

792

2898

        parse_values(&values)

793

2898

794

795

2922

    pub fn configured_channelling(&self) -> Result<Vec<Channelling>, String> {

796

2922

        let values = if self.channelling.is_empty() {

797

1248

            vec!["no".to_string()]

798

        } else {

799

1674

            self.channelling.clone()

800

};

801

2922

        let configured: Vec<Channelling> = parse_values(&values)?;

802

2922

        if configured.contains(&Channelling::Yes) {

803

            return Err("setting 'channelling=yes' is not supported yet".to_string());

804

2922

805

2922

        Ok(configured)

806

2922

807

808

    /// Resolve a backend's trace level, inheriting the fixture default when omitted.

809

4650

    pub fn configured_trace_level_for(&self, solver: SolverFamily) -> TraceLevel {

810

4650

        self.solver_options

811

4650

            .get(&solver.as_str())

812

1755

            .and_then(|options| options.trace_level)

813

4650

            .unwrap_or(self.trace_level)

814

4650

815

816

    /// Resolve a backend's heuristic while retaining the fixture default for other backends.

817

4650

    pub fn configured_heuristics_for(

818

4650

        &self,

819

4650

        solver: SolverFamily,

820

4650

    ) -> Result<Vec<Heuristic>, String> {

821

4650

        match self.solver_options.get(&solver.as_str()) {

822

1755

            Some(options) if !options.heuristic.is_empty() => parse_values(&options.heuristic),

823

2898

            _ => self.configured_heuristics(),

824

825

4650

826

827

    /// Resolve a backend's channelling policy while retaining the fixture default elsewhere.

828

4650

    pub fn configured_channelling_for(

829

4650

        &self,

830

4650

        solver: SolverFamily,

831

4650

    ) -> Result<Vec<Channelling>, String> {

832

4650

        let configured = match self.solver_options.get(&solver.as_str()) {

833

1755

            Some(options) if !options.channelling.is_empty() => parse_values(&options.channelling)?,

834

2922

            _ => self.configured_channelling()?,

835

};

836

4650

        if configured.contains(&Channelling::Yes) {

837

            return Err("setting 'channelling=yes' is not supported yet".to_string());

838

4650

839

4650

        Ok(configured)

840

4650

841

842

1773

    pub fn configured_solvers(&self) -> Result<Vec<SolverFamily>, String> {

843

1773

        for name in self.solver_options.keys() {

844

1755

            name.parse::<SolverFamily>()?;

845

846

1773

        parse_values(&self.solver)

847

1773

848

849

    pub fn uses_smt_solver(&self) -> bool {

850

        self.solver.iter().any(|solver| solver == "z3")

851

852

853

854

#[cfg(test)]

855

mod tests {

856

    use super::*;

857

858

    #[test]

859

1

    fn records_runs_in_the_canonical_shape() {

860

1

        let temp_dir = tempfile::tempdir().unwrap();

861

1

        let path = temp_dir.path().join(STATS_FILE_NAME);

862

1

        reset_stats_for_run(&path).unwrap();

863

1

        upsert_expected_time_stats(&path, 1).unwrap();

864

1

        upsert_status_stats(&path, "ok").unwrap();

865

1

        upsert_tool_status_stats(&path, "conjure", "ok").unwrap();

866

867

1

        upsert_conjure_timing_stats(

868

1

            &path,

869

1

            RecordedConjureStats {

870

1

                conjure_wall_clock_time: 0.572383542,

871

1

                conjure_translation_time: 0.572305542,

872

1

                conjure_driver_translation_time: 0.566305542,

873

1

                savilerow_translation_time: 0.006,

874

1

                conjure_solve_time: 0.000078,

875

1

},

876

1

877

1

        .unwrap();

878

879

1

        for (solver, translation_time) in [("minion", 1.0), ("sat-log", 2.0)] {

880

2

            let config = RecordedRunConfig {

881

2

                parser: "tree-sitter".to_string(),

882

2

                rewriter: "optimised".to_string(),

883

2

                comprehension_expander: "auto".to_string(),

884

2

                heuristic: "x".to_string(),

885

2

                channelling: "no".to_string(),

886

2

                seed: 0,

887

2

                solver_seed: 0,

888

2

                solver: solver.to_string(),

889

2

};

890

2

            let rows = (0..2)

891

                .map(|model_index| RecordedRunStats {

892

4

                    parser: config.parser.clone(),

893

4

                    rewriter: config.rewriter.clone(),

894

4

                    comprehension_expander: config.comprehension_expander.clone(),

895

4

                    heuristic: config.heuristic.clone(),

896

4

                    channelling: config.channelling.clone(),

897

4

                    solver: config.solver.clone(),

898

4

                    case_name: format!("tree-sitter-auto-model-{model_index:03}"),

899

4

                    model_index,

900

4

                    choice_path: vec![model_index],

901

4

                    choices: vec![format!("choice-{model_index}")],

902

4

                    status: "ok".to_owned(),

903

4

                    translation_time: Some(translation_time),

904

4

                    solve_time: Some(3.0),

905

4

                    rule_trace: RecordedRuleTraceStats {

906

4

                        total_rule_attempts: Some(4),

907

4

                        total_rule_applications: Some(model_index as u64 + 1),

908

4

                        rules: BTreeMap::from([("rule_name".to_owned(), model_index as u64 + 1)]),

909

4

},

910

4

                    ..RecordedRunStats::default()

911

4

})

912

                .collect();

913

2

            replace_config_run_stats(&path, &config, rows).unwrap();

914

915

916

1

        let contents = fs::read_to_string(&path).unwrap();

917

1

        let document: toml::Value = toml::from_str(&contents).unwrap();

918

1

        assert!(document.get("model-runs").is_none());

919

1

        assert_eq!(document["runs"].as_array().unwrap().len(), 4);

920

1

        assert_eq!(

921

1

            document["conjure"]["wall-clock-time"].as_float(),

922

1

            Some(0.572384)

923

);

924

1

        let stats = read_stats_or_default(&path).unwrap();

925

1

        assert_eq!(stats.expected_time, Some(1));

926

1

        for (index, run) in stats.runs.iter().enumerate() {

927

4

            assert_eq!(run.model_index, index % 2);

928

4

            assert_eq!(run.choice_path, vec![index % 2]);

929

4

            assert_eq!(

930

4

                run.rule_trace.total_rule_applications,

931

4

                Some((index % 2) as u64 + 1)

932

);

933

4

            assert_eq!(run.rule_trace.rules["rule_name"], (index % 2) as u64 + 1);

934

4

            assert_eq!(

935

4

                run.translation_time,

936

4

                Some(if index < 2 { 1.0 } else { 2.0 })

937

);

938

939

1

940

941

    #[test]

942

1

    fn replacing_a_portfolio_preserves_other_configurations_and_partial_failures() {

943

1

        let temp_dir = tempfile::tempdir().unwrap();

944

1

        let path = temp_dir.path().join(STATS_FILE_NAME);

945

1

        let config = RecordedRunConfig {

946

1

            parser: "tree-sitter".to_owned(),

947

1

            solver: "minion".to_owned(),

948

1

            rewriter: "optimised".to_owned(),

949

1

            comprehension_expander: "auto".to_owned(),

950

1

            heuristic: "x".to_owned(),

951

1

            channelling: "no".to_owned(),

952

1

            seed: 42,

953

1

            solver_seed: 7,

954

1

};

955

        let row = |model_index| RecordedRunStats {

956

4

            parser: config.parser.clone(),

957

4

            solver: config.solver.clone(),

958

4

            rewriter: config.rewriter.clone(),

959

4

            comprehension_expander: config.comprehension_expander.clone(),

960

4

            heuristic: config.heuristic.clone(),

961

4

            channelling: config.channelling.clone(),

962

4

            seed: config.seed,

963

4

            solver_seed: config.solver_seed,

964

4

            model_index,

965

4

            status: "ok".to_owned(),

966

4

            ..RecordedRunStats::default()

967

4

};

968

1

        replace_config_run_stats(&path, &config, vec![row(1), row(0)]).unwrap();

969

1

        let other_config = RecordedRunConfig {

970

1

            solver_seed: 8,

971

1

            ..config.clone()

972

1

};

973

1

        replace_config_run_stats(

974

1

            &path,

975

1

            &other_config,

976

1

            vec![RecordedRunStats {

977

1

                solver_seed: 8,

978

1

                ..row(0)

979

1

}],

980

1

981

1

        .unwrap();

982

1

        let mut failed = row(0);

983

1

        failed.status = "fail".to_owned();

984

1

        failed.translation_time = Some(1.25);

985

1

        replace_config_run_stats(&path, &config, vec![failed]).unwrap();

986

1

        let stats = read_stats_or_default(&path).unwrap();

987

1

        assert_eq!(stats.runs.len(), 2);

988

1

        assert_eq!(stats.runs[0].seed, 42);

989

1

        assert_eq!(stats.runs[0].solver_seed, 7);

990

1

        assert_eq!(stats.runs[0].status, "fail");

991

1

        assert_eq!(stats.runs[0].translation_time, Some(1.25));

992

1

        assert_eq!(stats.runs[0].solve_time, None);

993

1

        assert_eq!(stats.runs[1].solver_seed, 8);

994

1

        assert_eq!(stats.runs[1].status, "ok");

995

1

996

997

    #[test]

998

1

    fn resetting_stats_discards_the_previous_snapshot() {

999

1

        let temp_dir = tempfile::tempdir().unwrap();

1000

1

        let path = temp_dir.path().join(STATS_FILE_NAME);

1001

1

        fs::write(

1002

1

            &path,

1003

1

            "status = \"fail\"\n\n[[runs]]\ncomprehension-expander = \"native\"\nstatus = \"fail\"\n",

1004

1

1005

1

        .unwrap();

1006

1007

1

        let previous = read_stats_or_default(&path).unwrap();

1008

1

        assert_eq!(previous.runs[0].solver, "");

1009

1010

1

        reset_stats_for_run(&path).unwrap();

1011

1012

1

        assert_eq!(fs::read_to_string(path).unwrap(), "");

1013

1

1014