diff --git a/.cargo/config.toml b/.cargo/config.toml new file mode 100644 index 0000000..4d39d74 --- /dev/null +++ b/.cargo/config.toml @@ -0,0 +1,14 @@ +# Build flags for the bundled SQLite in tsq's differential oracle (rusqlite +# -> libsqlite3-sys reads this variable; sqawk itself does not use SQLite): +# +# * SQLITE_ENABLE_MATH_FUNCTIONS: ceil, floor, ... as in SQLite's own +# builds (the bundled amalgamation leaves them out). +# * LONGDOUBLE_TYPE=double: real <-> text conversion uses SQLite's +# double-double arithmetic on every platform, as MSVC builds always do and +# as current SQLite (3.50, which dropped long double) does everywhere, +# instead of x87 80-bit long double where the compiler offers it. Without +# this the oracle's 15th digit depends on the OS and on the x87 precision +# control word (CAST(100.0/476.42 AS TEXT) is ...535 with long double, +# ...536 without). +[env] +LIBSQLITE3_FLAGS = "-DSQLITE_ENABLE_MATH_FUNCTIONS -DLONGDOUBLE_TYPE=double" diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..bc3be01 --- /dev/null +++ b/.gitattributes @@ -0,0 +1,5 @@ +# Test fixtures are byte-exact: tests compare their bytes, writeback output +# and load-error line numbers. Never convert their line endings on checkout +# (core.autocrlf=true on Windows would make them CRLF). +tests/data/** -text +tests/*.csv -text diff --git a/CLAUDE.md b/CLAUDE.md index ab209cd..63532f9 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -29,8 +29,11 @@ cargo run -- -s "SELECT * FROM data" data.csv # Run in interactive REPL mode cargo run -- --interactive data.csv -# Generate test data with tsq (unpublished workspace crate in tsq/) -cargo run -p tsq -- --seed 42 --rows 1000 --output-dir /tmp/sqawk-test +# tsq (unpublished workspace crate in tsq/): data, differential testing +cargo run -p tsq -- gen --seed 42 --rows 1000 --output-dir /tmp/sqawk-test +cargo build --release --workspace +target/release/tsq diff --seed 1 --queries 20000 --keep-going --shrink +target/release/tsq corpus tsq/corpus/*.sql ``` ## Architecture Overview @@ -39,20 +42,31 @@ Sqawk is an SQL-based CLI tool for processing delimiter-separated files (CSV, TS ### Core Components -- **`main.rs`**: Entry point orchestrating the pipeline: CLI parsing → file loading → SQL execution → output → optional writeback -- **`database.rs`**: Central store for in-memory tables, manages table definitions and schemas -- **`table.rs`**: In-memory table representation with columns, rows, and projection capabilities -- **`sql_executor.rs`**: SQL parsing (via `sqlparser` crate) and execution against in-memory tables +- **`main.rs`**: Entry point orchestrating the pipeline: CLI parsing → file loading → SQL execution → output → optional writeback. Any error is printed once as `sqawk: ` (one line, exit 1); clap usage errors exit 2 +- **`cli.rs`**: clap arguments; `-s`/`-f` are merged into one ordered script by command-line position +- **`config.rs`**: `AppConfig`, the parsed options (-v, -F, --tabledef, --write, header/comment/output flags) passed to every component instead of global state +- **`repl.rs`**: the interactive shell (`-i`). On a terminal: rustyline editor with prompt, multi-line statements, tab completion and history (`$XDG_STATE_HOME/sqawk/history` when that is absolute, else `~/.sqawk_history`); piped, it reads lines and prints only command output. Dot commands (`.tables`, `.schema`, `.save`, `.cd`, `.load`, ...) live here +- **`error.rs`**: `SqawkError`; each `Display` is one complete line and no variant exposes a `source()` (that would print the cause twice). File I/O errors name the user's file (`SqawkError::open` / `write`), never a temp file +- **`diag.rs`**: `diag!` / `outln!` -- every stderr diagnostic (errors, `-v`) and REPL stdout line goes through them; write errors are ignored, so a closed stream never panics. Do not use `eprintln!`/`println!` +- **`database.rs`**: Central store for in-memory tables; owns modification tracking (`mark_modified` / `is_modified` / `modified_tables`, which decide what `--write` saves) and transactions (`begin` / `commit` / `rollback`; a table's original is kept the first time a transaction changes it) +- **`table.rs`**: In-memory table representation with columns (name, type affinity, declared type name), rows, and projection capabilities +- **`sql_executor.rs`**: `SqlExecutor`, the facade `main.rs` and the REPL use: `execute` hands SQL text to `vm::execute_vm` and applies the output delimiter to the result tables; it also saves modified tables (`save_modified_tables`, `save_table`), rolls back a transaction left open at exit (`finish`), and answers the REPL's table/column/row-count queries. It does no parsing itself ### SQL Execution (VM-based) SQL execution uses a bytecode VM inspired by SQLite's architecture: -1. SQL is parsed via the `sqlparser` crate +1. SQL is parsed by `src/vm/parse.rs` (`parse_sql`, sqlparser with `SqawkDialect`) 2. The AST is compiled to bytecode (`src/vm/compiler.rs`) 3. The VM engine executes the bytecode (`src/vm/engine.rs`) ### VM Module (`src/vm/`) +- `mod.rs`: `execute_vm` / `execute_statements`, the statement loop: parses + the script, then per statement runs BEGIN/COMMIT/ROLLBACK directly on the + `Database`, runs the pre-pass, compiles and executes the statement, applies + the engine's `TableModification`s (insert/replace/delete rows, create/drop/ + alter/truncate tables) to the `Database`, marking each changed table + modified, and drops the statement's registered tables - `bytecode.rs`: Bytecode instruction set and register/program types - `compiler.rs`: SQL AST to bytecode. Holds `code_expr`, the single expression compiler (the analogue of SQLite's `sqlite3ExprCode`) used from the SELECT @@ -81,18 +95,41 @@ SQL execution uses a bytecode VM inspired by SQLite's architecture: same query compiler as fragments of the enclosing program (`Once` when uncorrelated; correlation found by `NameCtx` outward resolution) - `compiler_check.rs`: rejects every parsed clause sqawk does not implement - (RETURNING, ON CONFLICT, UPDATE FROM, WITH, QUALIFY, ...) before + (RETURNING, ON CONFLICT, UPDATE FROM, QUALIFY, ...) before compilation; INSERT/UPDATE/DELETE/Query/Select are destructured exhaustively so a new sqlparser field fails to compile until handled +- `parse.rs`: `parse_sql`, the only SQL-to-AST entry point: parses with + `dialect.rs`'s `SqawkDialect` (Hive syntax, SQLite operator precedence), + rewrites compound SELECTs left-deep (equal precedence, left to right), and + enforces SQLite's limits (expression depth 1000, 500 compound terms) with an + iterative walk before anything recurses over the tree +- `prepass.rs`: runs before each statement compiles. Derived tables and CTEs + become registered tables (`sqawk_derived_N`, `sqawk_cte_N`) and FROM items + are rewritten to them. A non-recursive one is only compiled here and + registered lazily (`Database::add_lazy_table`); its rows are computed the + first time a cursor opens it (`OpenRead` → `VmEngine::fill_lazy`, SQLite's + `OP_Once`), so one read only by code that never runs is never computed. + Also CTE scoping (siblings, self, shadowing), `WITH ... DML` unwrapping, and + recursive CTEs, computed here (SQLite's queue algorithm, one run per + generation when the queue is FIFO). Other + one-part references sharing a CTE's name become `main.t`, and the compiler + refuses any one-part CTE name left over (`SqlCompiler::cte_names`), so a + missed rewrite cannot read the loaded table +- `parse.rs`: `parse_sql` (SQLite precedence, hex literals, MATERIALIZED + hints dropped, compounds left-deep, depth / compound limits) and + `for_each_child`, the one enumeration of a statement's queries, FROM items + and expressions shared by the limit walk and the pre-pass - `ast_compat.rs`: thin accessors normalizing sqlparser AST shapes - `engine.rs`: Executes bytecode instructions - `arith.rs`: SQLite numeric semantics -- text-to-number coercion, i64 - arithmetic with overflow to real, CAST, truth values + arithmetic with overflow to real, bitwise operators, CAST, truth values - `func.rs`: the scalar function table (`FuncDef`: name, arity, body) run by the `Function` opcode (SQLite's OP_Function); arity is checked at compile time. Lazy functions (COALESCE, IFNULL, IIF, NULLIF) are compiled to jumps -- `datetime.rs`: DATE/TIME/DATETIME/STRFTIME/... (SQLite time strings and - modifiers, UTC) +- `printf.rs`: SQLite's printf (a port of `sqlite3_str_vappendf`), used by + PRINTF/FORMAT, QUOTE and the date functions +- `datetime.rs`: DATE/TIME/DATETIME/STRFTIME/TIMEDIFF/... (a port of SQLite + 3.45's date.c: time strings, modifiers, `localtime` via the C library) - `like.rs`: the LIKE matcher (no regex; compiled pattern cached per instruction) - `agg.rs`: aggregate accumulators shared by AggStep/AggFinal and windows @@ -102,12 +139,19 @@ effects of the ones before it. ### File Handling -- **`file_handler.rs`**: Manages loading files into database tables +- **`file_handler.rs`**: Manages loading files into database tables and saving + them. `parse_table_spec` splits a file operand `[NAME=]PATH` (NAME only when + identifier-like; otherwise the table is named after the file stem; `-` is + stdin, table `stdin`); `has_tab_extension` is the `.tsv`/`.tab` rule used + for loading and for `CREATE TABLE ... LOCATION`; `load_location` loads (or + starts) a LOCATION table under its declared columns - **`csv_handler.rs`**: The single loader (`read_table`, built on the `csv` - crate) for every delimiter and for stdin, plus writeback. `.csv` files always - have a header; other files keep the header guess and `#`-comment skipping. - Fields load as Integer/Float only when the text round-trips exactly, so - untouched cells write back byte-for-byte + crate) for every delimiter and for stdin, plus writeback. `.tsv`/`.tab` are + tab-separated, everything else comma (unless `-F`); the first line is always + the header unless `--no-header`/`--tabledef`; `#` lines are comments only + under `--comments`. Each table keeps its file's bytes (`RawRows` in + `table.rs`: one buffer + per-row offsets), so untouched rows write back + byte-for-byte; every row mutation in `table.rs` keeps the offsets in sync ### SQL Features @@ -122,7 +166,7 @@ follow SQLite 3.45; deliberate differences are listed in ### Safe Writeback Model -By default, all modifications remain in memory only. The `--write` flag must be explicitly provided to save changes back to source files. Only tables modified by INSERT/UPDATE/DELETE are written. +By default, all modifications remain in memory only. The `--write` flag must be explicitly provided to save changes back to source files. Only tables changed by INSERT/UPDATE/DELETE/ALTER/TRUNCATE are written, and rows a statement did not change are written back byte for byte. `CREATE TABLE t (cols) LOCATION 'file'` follows Hive external-table semantics (`file_handler::load_location`): an existing file is loaded under the declared columns (its header line skipped, checked against the column count and written back verbatim; `WITH (HEADER=false)` makes the first line data; `WITH (COMMENTS=true)` is `--comments`) and loading it is not a modification; a file that does not exist yet is written even with no DML after it (created with just the header). The delimiter is WITH (DELIMITER=...) if given, else tab for `.tsv`/`.tab`, else comma. A LOCATION file already loaded as another table is an error. Scratch tables (CREATE TABLE without LOCATION, CREATE TABLE AS) are never written. Every target is checked for writability before any file is written; a read-only file aborts the whole writeback. An open transaction is rolled back before writeback. ## Test Organization @@ -130,7 +174,8 @@ Tests are in `tests/` organized by functionality: - `basic/`, `comparison/`, `join/`, `join_on/`, `order_by/`, `update/` - `unsupported/`: parsed-but-unimplemented syntax must error and modify nothing - `aggregate/`, `alias/`, `distinct/`, `group_by/`, `limit_offset/` -- `delimiter/`, `csv_handler/`, `string_functions/`, `repl/` +- `delimiter/`, `file_io/`, `headerless/`, `stdin/`, `string_functions/`, `repl/`, `transaction/` +- loader and writer unit tests live in `src/csv_handler.rs` (`#[cfg(test)]`) - `helpers/`: Test utilities - `common.rs`: Shared test helpers including REPL script runner @@ -145,10 +190,63 @@ Two suites carry the correctness contract: defects. Run with `cargo test --test mod defects -- --ignored`. Remove the `#[ignore]` when a defect is fixed; the list only shrinks. -`tests/sqlite_diff/harness.py` is a differential tester against SQLite (via -python3's `sqlite3` module), not part of `cargo test`. Run -`cargo build --release && python3 -I tests/sqlite_diff/harness.py -tests/sqlite_diff/queries.sql tests/sqlite_diff/queries2.sql`; the SUMMARY -line counts pass / DIFF (wrong answer) / sqawk_err. +`tsq/` (the unpublished `tsq` crate) is the differential tester and fuzzer. +Build with `cargo build --release --workspace`; `--sqawk PATH` or +`SQAWK=path` overrides the binary (default: the `sqawk` next to `tsq`). Data +goes to temp dirs. Module map: `rng.rs` (ChaCha8, the only randomness), +`data.rs` (tables, lossless CSV/TSV writer), `gen.rs` (`tsq gen`), `run.rs` +(spawning sqawk: time limit, stdin, cwd/env, stdout kept / drained / closed +early, Linux peak RSS from `/proc//status`), `corpus.rs`, `wb.rs` +(`tsq wb`), `clifuzz.rs` (`tsq cli`), `stress.rs` (`tsq stress`), and +`diff/`: `oracle.rs` +(rusqlite's bundled SQLite 3.45 -- `.cargo/config.toml` enables its math +functions -- loaded with sqawk's per-cell inference into untyped columns, +results rendered as sqawk prints them: reals via `CAST(x AS TEXT)`, NULL +empty, RFC 4180 quoting, headers included, compared byte for byte), +`schema.rs` (per-column properties deciding which comparisons both engines +agree on), `sqlgen.rs` (the generator, built on `ast.rs` shrinkable trees), +`allow.rs` (one rule per documented difference), `shrink.rs`, `tables.rs` +(built-in `harness` / `edge` table sets). + +- `tsq diff --seed N --queries 20000 --keep-going [--scale large] + [--tables edge] [--edge-rate 0.05] [--shrink --out DIR]`: random SQL + vs SQLite. `DIFF SUMMARY {pass, diff, sqawk_err, sqlite_err, both_err, + crash}`, then the documented-difference counts, SQLite time-outs (too + heavy for the reference; not findings) and `findings N` (exit 1 when + non-zero). `--only N` replays one test; `--print` shows the SQL. +- `tsq corpus tsq/corpus/*.sql`: the hand corpora (`queries*.sql` on the + `harness` tables, `hand*.sql` on `edge`, chosen by a `-- tsq-tables:` + line); same summary as `CORPUS SUMMARY`. +- `tsq wb --seed N --iterations 2000 [--only N] [--kill-rate F]`: + `--write` fuzzer. Random files (delimiters, LF/CRLF/mixed, BOM, ragged + rows, blank and `#` lines under `--comments`, quoted / multi-line / + invalid-UTF-8 cells, gen's edge values, `--no-header`, `CREATE TABLE ... + LOCATION` files) and random UPDATE / DELETE / INSERT / ALTER / TRUNCATE / + SELECT with BEGIN / COMMIT / END / ROLLBACK, run statement by statement on + the oracle too. Each file must equal the oracle's final table rendered by + the writeback rules (untouched rows = source bytes, tracked by rowid); + read-only targets refuse with nothing written; SELECT output and a + re-read must match; no stray files. Unix: SIGKILL (`kill -KILL`) during a + large write leaves old or new content (temp `.tmp*` leftovers are counted, + not failures). `WB SUMMARY {ok, bad, crash}`, must be bad 0 crash 0. +- `tsq cli --seed N --iterations 2000 [--only N]`: CLI / REPL surface fuzz + (flag and operand combinations, piped REPL scripts, the retired + csvfuzz.py's malformed files and mutations, the retired fuzz.py's garbage + SQL, limit values and huge statements via `-f`, closed stdout). Oracle: exit 0/1/2 (or `.exit + N`), no panic / overflow text, one `sqawk: ` line per error (clap's usage + format for 2), no hang, nothing created in cwd / HOME / XDG_STATE_HOME + (temp dirs), no banner or prompt in piped REPL output. `CLI SUMMARY {ok, + bad, crash}`, must be bad 0 crash 0. +- `tsq stress --rows 1000000 [--workloads a,b] [--json OUT] [--baseline FILE + --tolerance PCT] [--repeat K]`: wall time and (Linux) peak RSS per + workload; with `--baseline`, exit 1 on a regression. Not in CI. +- `cargo test --workspace` runs smoke budgets in `tsq/tests/tsq.rs`: `diff + --queries 300` (+150 on `edge`), `corpus` on all corpora, `wb --iterations + 50`, `cli --iterations 100` -- about 10 s serialized in a debug build. + +Every remaining DIFF / sqawk_err must be a difference or an unsupported +feature listed in `doc/sql_reference.md` ("Differences from SQLite", +"Unsupported syntax"); panic and hang must be 0. NULL and `''` both print +empty, so the testers cannot tell them apart. Tests use `assert_cmd` for CLI testing and `tempfile` for temporary test data. diff --git a/Cargo.lock b/Cargo.lock index a542a44..ecae6d3 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2,6 +2,18 @@ # It is not intended for manual editing. version = 4 +[[package]] +name = "ahash" +version = "0.8.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a15f179cd60c4584b8a8c596927aadc462e27f2ca70c04e0071964a73ba7a75" +dependencies = [ + "cfg-if", + "once_cell", + "version_check", + "zerocopy", +] + [[package]] name = "aho-corasick" version = "1.1.5" @@ -11,6 +23,15 @@ dependencies = [ "memchr", ] +[[package]] +name = "android_system_properties" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae221649c9976a6f6c56ae1facf410f3ddb33cc661c4b7b61020a912d4237fbc" +dependencies = [ + "libc", +] + [[package]] name = "anstream" version = "1.0.0" @@ -114,6 +135,12 @@ dependencies = [ "serde_core", ] +[[package]] +name = "bumpalo" +version = "3.20.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72f5acc6cb2ba439de613abc23857ec3d78374d8ed5ac84e9d11336e87da8649" + [[package]] name = "cc" version = "1.4.2" @@ -136,6 +163,17 @@ version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f079e83a288787bcd14a6aea84cee5c87a67c5a3e660c30f557a3d24761b3527" +[[package]] +name = "chrono" +version = "0.4.45" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1aa79e62e7697b8e29b513a68abacf485adcd1fe8284a4316c5ae868e6633327" +dependencies = [ + "iana-time-zone", + "num-traits", + "windows-link", +] + [[package]] name = "clap" version = "4.6.6" @@ -191,6 +229,12 @@ version = "1.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1d07550c9036bf2ae0c684c4297d503f838287c83c53686d05370d0e139ae570" +[[package]] +name = "core-foundation-sys" +version = "0.8.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" + [[package]] name = "csv" version = "1.4.0" @@ -240,6 +284,18 @@ version = "3.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dea2df4cf52843e0452895c455a1a2cfbb842a1e7329671acf418fdc53ed4c59" +[[package]] +name = "fallible-iterator" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2acce4a10f12dc2fb14a218589d4f1f62ef011b2d0cc4b3cb1bba8e94da14649" + +[[package]] +name = "fallible-streaming-iterator" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7360491ce676a36bf9bb3c56c1aa791658183a54d2744120f27285738d90465a" + [[package]] name = "fastrand" version = "2.5.0" @@ -272,6 +328,42 @@ dependencies = [ "num-traits", ] +[[package]] +name = "futures-core" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92d699e522242e69e3003b94ecc1f960f3a5e015aa7c5d7486e65ad01dd94f5e" + +[[package]] +name = "futures-task" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cd417de3d1d015fc3bfd2b1ea46dfc7bab72ef86f1cc7cc9c78e728b34a6d1fd" + +[[package]] +name = "futures-util" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0d50a92467f8ba5dd6e3ee5d4bd04d73ab2e4e1c44474a0674821dfce14b79bc" +dependencies = [ + "futures-core", + "futures-task", + "pin-project-lite", + "slab", +] + +[[package]] +name = "getrandom" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "899def5c37c4fd7b2664648c28120ecec138e4d395b459e5ca34f9cce2dd77fd" +dependencies = [ + "cfg-if", + "libc", + "r-efi 5.3.0", + "wasip2", +] + [[package]] name = "getrandom" version = "0.4.3" @@ -280,7 +372,25 @@ checksum = "300e883d756b2e4ec94e02791f39b04b522276138852cfc41d9fb7e904106099" dependencies = [ "cfg-if", "libc", - "r-efi", + "r-efi 6.0.0", +] + +[[package]] +name = "hashbrown" +version = "0.14.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e5274423e17b7c9fc20b6e7e208532f9b19825d82dfd615708b70edd83df41f1" +dependencies = [ + "ahash", +] + +[[package]] +name = "hashlink" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ba4ff7128dee98c7dc9794b6a411377e1404dba1c97deb8d1a55297bd25d8af" +dependencies = [ + "hashbrown", ] [[package]] @@ -298,6 +408,30 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "iana-time-zone" +version = "0.1.65" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e31bc9ad994ba00e440a8aa5c9ef0ec67d5cb5e5cb0cc7f8b744a35b389cc470" +dependencies = [ + "android_system_properties", + "core-foundation-sys", + "iana-time-zone-haiku", + "js-sys", + "log", + "wasm-bindgen", + "windows-core", +] + +[[package]] +name = "iana-time-zone-haiku" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f31827a206f56af32e590ba56d5d2d085f558508192593743f16b2306495269f" +dependencies = [ + "cc", +] + [[package]] name = "is_terminal_polyfill" version = "1.70.2" @@ -310,12 +444,34 @@ version = "1.0.18" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" +[[package]] +name = "js-sys" +version = "0.3.105" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ce57d20d1ea864ce2ac172ab472d409214f4fd359f0b2a2775abdf522e2af99e" +dependencies = [ + "cfg-if", + "futures-util", + "wasm-bindgen", +] + [[package]] name = "libc" version = "0.2.189" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3eaf3ede3fee6db1a4c2ee091bf8a8b4dccdc6d17f656fb07896ee72867612f2" +[[package]] +name = "libsqlite3-sys" +version = "0.28.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c10584274047cb335c23d3e61bcef8e323adae7c5c8c760540f73610177fc3f" +dependencies = [ + "cc", + "pkg-config", + "vcpkg", +] + [[package]] name = "linux-raw-sys" version = "0.12.1" @@ -391,6 +547,27 @@ version = "1.70.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "384b8ab6d37215f3c5301a95a4accb5d64aa607f1fcb26a11b5303878451b4fe" +[[package]] +name = "pin-project-lite" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a89322df9ebe1c1578d689c92318e070967d1042b512afbe49518723f4e6d5cd" + +[[package]] +name = "pkg-config" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f6b464fbc74e149a392436b17d523f769e057cb6877f6a5c4618bc6f11800548" + +[[package]] +name = "ppv-lite86" +version = "0.2.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85eae3c4ed2f50dcfe72643da4befc30deadb458a9b590d720cde2f2b1e97da9" +dependencies = [ + "zerocopy", +] + [[package]] name = "predicates" version = "3.1.4" @@ -449,6 +626,12 @@ dependencies = [ "proc-macro2", ] +[[package]] +name = "r-efi" +version = "5.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "69cdb34c158ceb288df11e18b4bd39de994f6657d83847bdffdbd7f346754b0f" + [[package]] name = "r-efi" version = "6.0.0" @@ -465,6 +648,35 @@ dependencies = [ "nibble_vec", ] +[[package]] +name = "rand" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9ef1d0d795eb7d84685bca4f72f3649f064e6641543d3a8c415898726a57b41" +dependencies = [ + "rand_chacha", + "rand_core", +] + +[[package]] +name = "rand_chacha" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3022b5f1df60f26e1ffddd6c66e8aa15de382ae63b3a0c1bfc0e4d3e3f325cb" +dependencies = [ + "ppv-lite86", + "rand_core", +] + +[[package]] +name = "rand_core" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "76afc826de14238e6e8c374ddcc1fa19e374fd8dd986b0d2af0d02377261d83c" +dependencies = [ + "getrandom 0.3.4", +] + [[package]] name = "recursive" version = "0.1.1" @@ -514,6 +726,20 @@ version = "0.8.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d6f6ff9a378485b298a5286656da665ba74413d36db0979633275d2e708145d4" +[[package]] +name = "rusqlite" +version = "0.31.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b838eba278d213a8beaf485bd313fd580ca4505a00d5871caeb1457c55322cae" +dependencies = [ + "bitflags", + "fallible-iterator", + "fallible-streaming-iterator", + "hashlink", + "libsqlite3-sys", + "smallvec", +] + [[package]] name = "rustix" version = "1.1.4" @@ -527,6 +753,12 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "rustversion" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf54715a573b99ac80df0bc206da022bcd442c974952c7b9720069370852e21f" + [[package]] name = "rustyline" version = "15.0.0" @@ -581,6 +813,12 @@ version = "2.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f8fadd59c855ef2080decdef8ff161eb6661b86933c9d82e5ba29dc602a55aba" +[[package]] +name = "slab" +version = "0.4.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c790de23124f9ab44544d7ac05d60440adc586479ce501c1d6d7da3cd8c9cf5" + [[package]] name = "smallvec" version = "1.15.2" @@ -589,16 +827,16 @@ checksum = "8ed6a63f02c8539c91a8685a86f4099661ba3da017932f6ebbea6de3f0fa7c90" [[package]] name = "sqawk" -version = "0.8.2" +version = "0.9.0" dependencies = [ - "anyhow", "assert_cmd", + "chrono", "clap", "csv", "predicates", - "regex", "rustyline", "sqlparser", + "stacker", "tempfile", ] @@ -660,7 +898,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd" dependencies = [ "fastrand", - "getrandom", + "getrandom 0.4.3", "once_cell", "rustix", "windows-sys 0.61.2", @@ -674,12 +912,14 @@ checksum = "8f50febec83f5ee1df3015341d8bd429f2d1cc62bcba7ea2076759d315084683" [[package]] name = "tsq" -version = "0.1.0" +version = "0.2.0" dependencies = [ "anyhow", "assert_cmd", "clap", - "libc", + "rand", + "rand_chacha", + "rusqlite", "tempfile", ] @@ -707,6 +947,18 @@ version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" +[[package]] +name = "vcpkg" +version = "0.2.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426" + +[[package]] +name = "version_check" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" + [[package]] name = "wait-timeout" version = "0.2.1" @@ -716,12 +968,119 @@ dependencies = [ "libc", ] +[[package]] +name = "wasip2" +version = "1.0.4+wasi-0.2.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b67efb37e106e55ce722a510d6b5f9c17f083e5fc79afc2badeb12cc313d9487" +dependencies = [ + "wit-bindgen", +] + +[[package]] +name = "wasm-bindgen" +version = "0.2.128" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aecb87a33d3b0c5e3b7aa46336eaf486cffafbd281b195e4c8b80d50df2351bf" +dependencies = [ + "cfg-if", + "once_cell", + "rustversion", + "wasm-bindgen-macro", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-macro" +version = "0.2.128" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a690d511e3c1a8b3a55e33511e3c2c00c78415cd23650f32b808627f5696b9ed" +dependencies = [ + "quote", + "wasm-bindgen-macro-support", +] + +[[package]] +name = "wasm-bindgen-macro-support" +version = "0.2.128" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "411e4887f0071ef2d2164a9d5fdf2d20efbef78fccd3a78b0c10a1dc5295e48a" +dependencies = [ + "bumpalo", + "proc-macro2", + "quote", + "syn 3.0.3", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-shared" +version = "0.2.128" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "81941cd78d0c92026c33e5e01312845a4cb1e9af3407f9134b100dd03144103e" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "windows-core" +version = "0.62.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8e83a14d34d0623b51dce9581199302a221863196a1dde71a7663a4c2be9deb" +dependencies = [ + "windows-implement", + "windows-interface", + "windows-link", + "windows-result", + "windows-strings", +] + +[[package]] +name = "windows-implement" +version = "0.60.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "053e2e040ab57b9dc951b72c264860db7eb3b0200ba345b4e4c3b14f67855ddf" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "windows-interface" +version = "0.59.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f316c4a2570ba26bbec722032c4099d8c8bc095efccdc15688708623367e358" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + [[package]] name = "windows-link" version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" +[[package]] +name = "windows-result" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7781fa89eaf60850ac3d2da7af8e5242a5ea78d1a11c49bf2910bb5a73853eb5" +dependencies = [ + "windows-link", +] + +[[package]] +name = "windows-strings" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7837d08f69c77cf6b07689544538e017c1bfcf57e34b4c0ff58e6c2cd3b37091" +dependencies = [ + "windows-link", +] + [[package]] name = "windows-sys" version = "0.59.0" @@ -803,3 +1162,29 @@ name = "windows_x86_64_msvc" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" + +[[package]] +name = "wit-bindgen" +version = "0.57.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ebf944e87a7c253233ad6766e082e3cd714b5d03812acc24c318f549614536e" + +[[package]] +name = "zerocopy" +version = "0.8.62" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "86502bf56ac7c77571a32e2647bb2a15894565e981fb2a48d7bde2d91c965a9d" +dependencies = [ + "zerocopy-derive", +] + +[[package]] +name = "zerocopy-derive" +version = "0.8.62" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5457206954b06561e2608c7e19cf58b1926586d999c246eebe4502f7e2039d1a" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] diff --git a/Cargo.toml b/Cargo.toml index 1aae302..85ea1aa 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "sqawk" -version = "0.8.2" +version = "0.9.0" edition = "2021" authors = ["Jeff Garzik "] description = "An SQL-based command-line tool for processing delimiter-separated files (CSV, TSV, etc.), inspired by awk" @@ -9,7 +9,7 @@ repository = "https://github.com/jgarzik/sqawk" homepage = "https://github.com/jgarzik/sqawk" readme = "README.md" keywords = ["csv", "tsv", "sql", "delimited", "awk"] -categories = ["command-line-utilities", "text-processing", "parser-implementations"] +categories = ["command-line-utilities", "text-processing"] # Minimum supported Rust version. # # Verified by building with that toolchain, not inferred. The binding @@ -31,11 +31,15 @@ exclude = [ clap = { version = "4", features = ["derive"] } sqlparser = "0.62" csv = "1" -anyhow = "1" -regex = "1" tempfile = "3" rustyline = "15" +# Runs sqawk on a large stack (see MAIN_STACK_BYTES in main.rs); already in +# the build through sqlparser -> recursive. +stacker = "0.1" +# The local time zone for the `localtime` and `utc` date modifiers ($TZ, +# else the system zone). +chrono = { version = "0.4", default-features = false, features = ["clock", "std"] } [dev-dependencies] assert_cmd = "2" @@ -43,10 +47,13 @@ predicates = "3" [profile.release] lto = true +codegen-units = 1 +strip = true [profile.profiling] inherits = "release" debug = true +strip = false [[bin]] name = "sqawk" diff --git a/README.md b/README.md index 398cf17..359de9a 100644 --- a/README.md +++ b/README.md @@ -8,27 +8,34 @@ Sqawk is an SQL-based command-line tool for processing delimiter-separated files ## Features - **SQL Query Engine** - SELECT, INSERT, UPDATE, DELETE with WHERE, ORDER BY, GROUP BY, HAVING, LIMIT/OFFSET -- **Joins** - INNER, LEFT, RIGHT, FULL OUTER, and CROSS joins with ON conditions +- **Joins** - INNER, LEFT, RIGHT, FULL OUTER, and CROSS joins with ON, USING or NATURAL; equality joins run as hash joins - **Aggregates** - COUNT, SUM, TOTAL, AVG, MIN, MAX, GROUP_CONCAT/STRING_AGG with GROUP BY, DISTINCT and `FILTER (WHERE ...)` -- **Functions** - String (UPPER, LOWER, LENGTH, SUBSTR, INSTR, REPLACE, TRIM, LPAD/RPAD, PRINTF, ...), math (ABS, ROUND, CEIL, FLOOR, SIGN, multi-argument MIN/MAX), conditional (COALESCE, IFNULL, NULLIF, IIF, TYPEOF), date/time (DATE, TIME, DATETIME, STRFTIME, JULIANDAY, UNIXEPOCH with modifiers) +- **Functions** - String (UPPER, LOWER, LENGTH, SUBSTR, INSTR, REPLACE, TRIM, LPAD/RPAD, PRINTF, QUOTE, ...), math (ABS, ROUND, CEIL, FLOOR, SIGN, multi-argument MIN/MAX, bitwise operators), conditional (COALESCE, IFNULL, NULLIF, IIF, TYPEOF), date/time (DATE, TIME, DATETIME, STRFTIME, JULIANDAY, UNIXEPOCH, TIMEDIFF with SQLite's modifiers) - **SQLite semantics** - 64-bit integer arithmetic, SQLite CAST rules, ASCII case-insensitive LIKE with ESCAPE, character-based strings; unimplemented syntax is an error, never silently ignored - **Subqueries** - Scalar, `IN (SELECT ...)`, and `EXISTS`, including correlated +- **Common Table Expressions** - `WITH` and `WITH RECURSIVE` (tree walks, counters), on SELECT, INSERT, UPDATE and DELETE - **Set Operations** - UNION, UNION ALL, INTERSECT, EXCEPT - **Window Functions** - ROW_NUMBER, RANK, DENSE_RANK, PERCENT_RANK, CUME_DIST, NTILE, LAG, LEAD, FIRST_VALUE, LAST_VALUE, NTH_VALUE, and aggregates with `OVER (PARTITION BY ... ORDER BY ...)`, ROWS frames and named windows - **DDL** - CREATE TABLE, CREATE TABLE AS SELECT, DROP, ALTER TABLE ADD COLUMN, TRUNCATE +- **Transactions** - BEGIN, COMMIT, ROLLBACK over the in-memory tables; nothing uncommitted is written - **Expressions** - CASE, CAST, BETWEEN, IN, LIKE/ILIKE ... ESCAPE, IS [NOT] DISTINCT FROM, `||`, arithmetic; ORDER BY ... NULLS FIRST/LAST -- **File Formats** - CSV, TSV, and custom delimiters; headerless files via `--no-header` or `--tabledef`; unchanged cells survive `--write` byte for byte +- **File Formats** - CSV, TSV (`.tsv`/`.tab`), and custom delimiters (`-F`); headerless files via `--no-header` or `--tabledef`; `#` comment lines via `--comments`; untouched rows survive `--write` byte for byte - **Pipelines** - Reads standard input with a `-` file operand - **Safe by Default** - Files unchanged unless `--write` flag is specified - **Interactive REPL** - Explore data interactively with `-i` flag +Every input file is loaded whole into memory: plan on six to seven times the +file's size, plus the result set (see +[Working with Large Files](doc/user_guide.md#working-with-large-files)). + ## Installation ```sh -cargo install sqawk +cargo install --locked sqawk ``` -This installs the `sqawk` binary. Requires Rust 1.88 or newer. +This installs the `sqawk` binary. Requires Rust 1.88 or newer. `--locked` +builds with the dependency versions sqawk was tested with. ## Quick Examples @@ -47,32 +54,87 @@ sqawk -s "UPDATE data SET status = 'archived' WHERE year < 2020" data.csv --writ # Read standard input as the table "stdin" cat employees.csv | sqawk -s "SELECT name FROM stdin WHERE salary > 70000" - + +# Run the statements in a file; print tab-separated results without a header +sqawk -f report.sql --output-delimiter '\t' --no-output-header employees.csv ``` -## `tsq` - test data generator (development only) +Errors are one line on stderr (`sqawk: no such column: nmae`) with exit +status 1; usage errors exit with status 2. + +## `tsq` - test data generator, differential tester and fuzzer (development only) -The source tree also contains `tsq`, which generates deterministic multi-table -CSV data plus a corpus of SQL queries for exercising sqawk. It is a separate -workspace crate (`tsq/`) that is not published, so `cargo install sqawk` does -not install it. Run it from a checkout: +The source tree also contains `tsq`, the tool that tests sqawk: it generates +deterministic test data, checks sqawk's answers and its `--write` output +against SQLite, fuzzes the command line and the REPL, and measures +performance. It is a separate workspace crate (`tsq/`) that is not +published, so `cargo install sqawk` does not install it. Run it from a +checkout: ```sh -cargo run -p tsq -- --seed 42 --rows 1000 --output-dir /tmp/sqawk-test +cargo build --release --workspace +target/release/tsq gen --seed 42 --rows 1000 --output-dir /tmp/sqawk-test sqawk -s "SELECT * FROM customers LIMIT 10" /tmp/sqawk-test/data/customers.csv +target/release/tsq diff --seed 1 --queries 20000 --keep-going --shrink +target/release/tsq corpus tsq/corpus/*.sql +target/release/tsq wb --seed 1 --iterations 2000 +target/release/tsq cli --seed 1 --iterations 2000 +target/release/tsq stress --rows 1000000 --json stress.json ``` -It writes `data/` (customers, products, orders, order_items, reviews, with -realistic foreign-key relationships), `queries/` (numbered `.sql` files -covering selects, joins, aggregates, subqueries and window functions), -`verify/run_verification.sh`, and a `metadata.json` recording the seed and row -counts. The same seed always produces the same data. - -| Option | Meaning | -| --- | --- | -| `-s`, `--seed` | Seed for reproducible generation; a random one is printed if omitted | -| `-r`, `--rows` | Base customer row count; other tables scale proportionally (default 100000) | -| `-o`, `--output-dir` | Where to write the generated tree (required) | -| `-v`, `--verbose` | Show generation progress | +The global options `--seed` (64-bit; a clock seed is printed when omitted), +`--sqawk PATH` (default: `$SQAWK`, else the `sqawk` next to `tsq`) and +`--verbose` apply to every subcommand. The same seed gives the same data and +queries on every OS. + +- **`tsq gen`** writes `data/` (customers, products, orders, order_items, + reviews, with foreign keys and realistic distributions) and a + `metadata.json` recording the seed, options and row counts. `--rows N` + (customers; the other tables scale from it), `--edge-rate F` (replace + non-key cells with edge values: i64 limits, `007`, `1.50`, ` 5`, Unicode, + embedded quotes and line breaks, long strings, ...), `--format csv|tsv`, + `--crlf`, `--bom`, `--no-header`, `--ragged-rate F` (rows that leave off + trailing empty fields). +- **`tsq diff`** generates random SQL (expressions, joins of every kind, + subqueries, aggregates, windows, compounds, CTEs including recursive ones, + DML sequences, transactions, `CREATE TABLE AS`, `ALTER TABLE`) and runs + each test through sqawk and through SQLite 3.45 loaded with the same + values, comparing every result set byte for byte. `--queries N`, + `--scale small|large|ROWS`, `--tables gen|edge|harness`, `--edge-rate F`, + `--keep-going`, `--shrink` (minimize each finding; reproductions go to + `--out DIR`), `--only N` (one test), `--print`, `--jobs N`, `--timeout S`. + The last line is `DIFF SUMMARY {pass, diff, sqawk_err, sqlite_err, + both_err, crash}`; non-passing results covered by a documented difference + are counted separately, and any other is a finding (exit status 1). +- **`tsq corpus FILE...`** runs hand-written query files (one test per line, + `O:` for an ordered comparison) the same way; `tsq/corpus/` holds them. +- **`tsq wb`** fuzzes `--write`: random files (CSV / TSV / `-F` separators, + LF / CRLF / mixed line endings, byte order marks, ragged rows, blank and + `#` comment lines, quoted and multi-line fields, edge values) and random + UPDATE / DELETE / INSERT / ALTER TABLE / TRUNCATE sequences with + transactions and `CREATE TABLE ... LOCATION`. Every file must equal what + the writeback rules give for SQLite's final table (untouched rows byte for + byte, changed rows from their values); rolled-back work and read-only + targets must leave files untouched. On Unix it also kills sqawk during a + large write: the file must be the old or the new content, never partial. + `--iterations N`, `--only N`, `--kill-rate F`. Last line: `WB SUMMARY + {ok, bad, crash}`. +- **`tsq cli`** fuzzes the command line and the REPL: random flag and + operand combinations, piped REPL scripts, malformed files, garbage and + very large SQL, standard output closed early. Every run must exit 0, 1 or + 2 (or a `.exit N` status) within the time limit, print errors as one + `sqawk: ` line, never panic, and leave no stray files. `--iterations N`, + `--only N`. Last line: `CLI SUMMARY {ok, bad, crash}`. +- **`tsq stress`** times workloads (scan, filter, group, joins, sort, + distinct, windows, CTEs, subqueries, compounds, `--write`, a 1000-column + table) over generated data, `--rows` 1,000,000 to 10,000,000 in the + largest table, recording wall time and, on Linux, the peak RSS of sqawk. + `--json OUT` saves the results; `--baseline FILE --tolerance PCT` exits 1 + when a workload regressed; `--repeat K` keeps the best of K runs; + `--workloads a,b` / `--list`. + +`cargo test --workspace` runs small budgets of `diff`, `corpus`, `wb` and +`cli`; `stress` is run by hand. ## Documentation diff --git a/doc/database.md b/doc/database.md index 1c932c2..ced957b 100644 --- a/doc/database.md +++ b/doc/database.md @@ -9,8 +9,10 @@ Sqawk implements a lightweight in-memory database engine that supports SQL opera 1. [Data Model](#data-model) 2. [Architecture](#architecture) 3. [File I/O Patterns](#file-io-patterns) -4. [Storage Backend Architecture](#storage-backend-architecture) +4. [Storage](#storage) 5. [Performance Considerations](#performance-considerations) +6. [Join Implementation](#join-implementation) +7. [Current System Limitations](#current-system-limitations) ## Data Model @@ -19,39 +21,105 @@ Sqawk implements a lightweight in-memory database engine that supports SQL opera Tables in Sqawk are in-memory representations of delimiter-separated files. Each table has: - A unique name (derived from the file's filename or explicitly provided) -- A set of columns with names (derived from the file's header row) +- A set of named columns (see [Where Columns Come From](#where-columns-come-from)) - Zero or more rows of data - Optional metadata including the source file path and delimiter information -Tables maintain an internal mapping of column names to their indices for efficient access. +Column names are looked up case-insensitively (`Table::column_index`), the +one lookup used by SELECT, INSERT column lists and UPDATE SET. ### Rows and Columns - **Rows**: Represented as vectors of values with one element per column - **Columns**: Identified by name, with automatic type inference based on content -- **Schema**: Dynamically determined from the file's header row -- **Column Types**: Not declared; inferred at runtime **per value**, so a single - column may hold integers, floats and text +- **Schema**: The column names come from one of the sources listed under + [Where Columns Come From](#where-columns-come-from) +- **Column Types**: A loaded column has no declared type; a column created by + CREATE TABLE or ALTER TABLE ADD COLUMN records its declared type name and + that name's SQLite affinity (`Column::decl_type`, `Column::data_type`), but + never converts a value. Types live **per value**, so a single column may + hold integers, floats and text - **NULL**: An empty field reads as NULL. Comparisons follow SQL three-valued logic, so a comparison involving NULL is UNKNOWN and its row is filtered out rather than raising an error. Sorting and grouping treat NULL as equal to NULL, which SQL comparison must not — the two use separate paths deliberately -- **Coercion**: A comparison mixing a number and a string converts the string - when it parses as a number, and compares textually otherwise. See - [SQL Reference](sql_reference.md) for the exact rules +- **Comparison**: One rule everywhere (`=`, `<`, IN lists and subqueries, + joins, ORDER BY, MIN/MAX): two values that both read as numbers -- numbers, + or text spelling a number such as `'007'` or `'1e3'` -- compare as numbers, + text against text included (`'10' > '9'`); numbers sort before other text; + other text compares byte by byte. DISTINCT, GROUP BY and the compound + operators still keep text `'1'` and integer `1` apart. See + [SQL Reference](sql_reference.md#type-coercion) for the exact rules + +### Where Columns Come From + +A table's columns are defined in exactly one of these ways: + +1. **The file's header line** (the default for an input file): the first + line names the columns and is not data. On `--write` it is written back + as it was read. +2. **`--no-header`**: every line of every input file is data and the + columns are named `a`, `b`, `c`, ...; writeback emits no header line. +3. **`--tabledef=table:col1,col2,...`**: the names are declared before the + file loads, and that file's first line is data (as with `--no-header`). + `--tabledef` takes names only, never types. +4. **`CREATE TABLE t (col TYPE, ...) LOCATION 'path'`**: the declared names + and types are the schema of the file, as for a Hive external table. + - When the file exists it is loaded under the declared names. Its first + line is a header by default: it is skipped as data, must have exactly + one field per declared column (else an error naming the file and both + counts), and is written back verbatim -- the declared names do not + replace its text. With `WITH (HEADER=false)` the first line is data and + writeback emits no header. + - When the file does not exist the table starts empty and `--write` + creates the file, with a header line of the declared names (none under + `HEADER=false`). +5. **`CREATE TABLE t AS SELECT ...`**: the query's result columns. The + table has no file and is never written. + +`CREATE TABLE t (col TYPE, ...)` without `LOCATION` declares the columns of +a scratch table, which has no file and is never written. Declared types are +recorded (`.schema` shows them) but not enforced: every value keeps its own +type. ### Table Lifecycle Tables are loaded from delimiter-separated files at startup and can be modified through SQL operations. Sqawk uses a safe, sed-like writeback model: 1. **Default Behavior**: All modifications remain in memory only -2. **Tracking Changes**: The system tracks which tables have been modified by any operation +2. **Tracking Changes**: The database keeps the set of modified tables + (`Database::modified`, set by `Database::mark_modified`): INSERT, UPDATE, + DELETE, ALTER TABLE and TRUNCATE mark a table, CREATE TABLE marks the new + one unless its `LOCATION` file already exists (so a new `CREATE TABLE ... + LOCATION` file is written even with no rows, while loading an existing + one is not a modification), and DROP TABLE removes the mark with the + table 3. **Write on Exit**: Modified tables are only written back to their source files if: - The `--write` (or `-w`) flag is explicitly provided - - The table was actually modified by an SQL operation (INSERT, UPDATE, DELETE) + - The table was actually modified by one of those statements + - The table has a file: a scratch table (CREATE TABLE without LOCATION, + CREATE TABLE ... AS) is skipped. A table read from standard input is an + error instead 4. **Safe Execution**: Without the `--write` flag, source files remain untouched regardless of operations performed 5. **Write Only Modified**: Only tables that were changed are written; unmodified tables are not rewritten -6. **Format Preservation**: Original file formats and delimiters are preserved during writeback +6. **Check First**: Every target is checked before any file is written; a + read-only file, or a directory that cannot take the temp file, fails the + run with no file written +7. **Format Preservation**: Original file formats and delimiters are preserved during writeback + +### Transactions + +`BEGIN` / `COMMIT` (`END`) / `ROLLBACK` are handled in `vm/mod.rs` before +compilation and act on the `Database` directly. The undo state is +copy-on-write: `BEGIN` records only the modified-table set; the first +`get_table_mut` of a table inside the transaction stores a deep copy of it +(rows, source bytes, layout), `remove_table` moves the dropped table into +the undo state, and `add_table` records that the name did not exist. +`ROLLBACK` puts every recorded table back and restores the modified set; +`COMMIT` discards the undo state. A transaction that only reads copies +nothing, and one that changes a table copies it once. A transaction still +open at the end of the run (or REPL session) is rolled back before writeback. +Savepoints are not supported. This design ensures that users can experiment with data manipulations while maintaining the integrity of source files. The verbose mode (`-v`) provides additional confirmation about whether changes were saved or not. @@ -65,20 +133,26 @@ The `Table` struct represents an in-memory table with: - Column metadata - Row data - Methods for accessing and manipulating rows -- Projection capabilities (selecting subsets of columns) +- The source file's bytes and layout, for verbatim writeback (see Storage) ### SQL Execution: a bytecode VM SQL is not interpreted against the tables directly. It is compiled to bytecode and run on a register virtual machine, following SQLite's architecture: -1. **Parse** — the `sqlparser` crate produces an AST +1. **Parse** — `vm::parse::parse_sql` produces an AST (the `sqlparser` crate + with `SqawkDialect`) 2. **Compile** — `vm::compiler` lowers the AST to a bytecode program 3. **Execute** — `vm::engine` runs that program against table cursors -`SqlExecutor` is a thin wrapper over this pipeline: it forwards to the VM, -tracks which tables were modified, and provides `save_modified_tables()`, which -writes back only tables that actually changed. +Modification tracking belongs to `Database`: `vm::execute_vm` applies each +statement's changes and calls `database.mark_modified(name)` for every table +it changed, and `Database::is_modified` / `modified_tables` report the set. +Code that changes a table outside that path must call +`database.mark_modified` itself, or `--write` will not save the table. +`SqlExecutor` is a thin facade over the pipeline: it forwards SQL to +`vm::execute_vm`, and its `save_modified_tables()` writes back the tables in +`Database::modified_tables()` that have a file. The compiler has one recursive expression compiler, `code_expr` (the analogue of SQLite's `sqlite3ExprCode`), used from every position an expression can @@ -151,7 +225,12 @@ false even for a NULL operand). only; a correlated one reruns on every evaluation, i.e. per outer row. - **Correlated execution**: a subquery's cursors and hash indexes do not depend on the outer row, so they are set up once (`Once` around the - `OpenRead`s and hash builds). An equality between an inner column and an + `OpenRead`s and hash builds). In any loop nest, an inner level's table is + opened and its hash index built when the nest first reaches that level + (a `Once` there), so a join whose outer loop has no rows never reads its + inner tables -- a CTE or derived table there is never computed. (Not in a + FULL JOIN, nor for a RIGHT JOIN group's inner levels, which can be + NULL-extended unreached.) An equality between an inner column and an outer column -- `WHERE d.k = f.k` -- becomes a hash probe, so a correlated `EXISTS`/`IN`/scalar lookup costs one probe per outer row instead of a rescan. Aggregate accumulators, sorters and DISTINCT tables are reset at @@ -170,22 +249,83 @@ consequences follow: - A statement observes the database as the previous one left it, which is what `CREATE TABLE t; INSERT INTO t ...; SELECT * FROM t` requires. -A subquery in `FROM` (a derived table) is materialized before its enclosing -statement compiles: the subquery runs, its result is registered under the -alias, and the statement is rewritten to reference that name. The registration -lasts only for that statement. +### Derived Tables and CTEs: the pre-pass + +The compiler only opens cursors on named tables. Before a statement compiles, +a pre-pass (`src/vm/prepass.rs`) turns every derived table and CTE reference +into one. It walks the statement with `parse::for_each_child` -- the same +enumeration of queries, FROM items and expressions the parser's limit check +uses (CTE bodies, the statement inside `WITH ... INSERT`, JOIN ON, GROUP BY, +window specs, function arguments, LIMIT, UPDATE SET, ...) -- and for each FROM +item that is: + +- a **derived table** `(SELECT ...) d`: walks its query (innermost first), + compiles it, registers the result as `sqawk_derived_N` and rewrites the + item to that table with alias `d`; +- a **one-part name of a CTE in scope**: compiles the CTE on its first + reference (every other reference reuses the table) as `sqawk_cte_N`, and rewrites + the item to it, aliased as written or else by the CTE's name -- so `c.col` + resolves and the internal name never appears in output or errors; +- **any other one-part name that a CTE of the statement declares**: rewrites + it to `main.name`, the loaded table. + +A registered derived table or (non-recursive) CTE is a *lazy table* +(`Database::add_lazy_table`): its columns come from the compiled query and +its rows from running that program the first time an `OpenRead` opens it +(`VmEngine::fill_lazy`), as SQLite runs a FROM subquery behind `OP_Once` +where it is first read. Since the FROM planner tests WHERE terms that read +no table before opening any cursor, a CTE read only under a false WHERE -- +or only in a CASE branch not taken -- is never run. A recursive CTE is +computed in the pre-pass. + +CTE scopes are a stack of WITH lists. A CTE is Pending, InProgress (being +computed: a reference now is `circular reference`) or Done. Its body is +walked with the stack cut back to its own WITH list, so it sees its siblings +and itself but not the scopes around the reference. The WITH is removed from +the query; an unused CTE's body is never walked or run; `WITH ... DML` +becomes the DML statement. + +**Guard**: the compiler receives the set of CTE names declared in the +statement (`SqlCompiler::cte_names`). Since the pre-pass rewrote every +reference it reached -- to an internal name or to `main.name` -- a one-part +table name from that set reaching `get_table_name` is a reference the walk +missed, and is the error `internal error: unresolved CTE c` instead of a +silent read of the loaded table of that name. + +**Recursive CTEs** run in Rust, as SQLite does: the setup arms run as one +query whose rows seed a queue; recursive arms are compiled once, with their +self-reference rewritten to a registered working table; each row taken from +the queue is output (OFFSET / LIMIT) and becomes the working table's content +for a run of each arm's program, whose rows join the queue (UNION: only rows +not seen before, by the compound row key). The queue is FIFO, or a stable +heap ordered by the CTE body's ORDER BY. A FIFO queue is breadth-first, so +when no arm has DISTINCT or a wildcard, a whole generation is loaded into +the working table at once with a position column, and the children are put +back in queue order by (position, arm): the same rows in the same order, one +run per generation instead of per row. + +Everything registered lasts until the statement ends, success or failure. +A derived table or CTE body is computed up front, so one that reads a column +of an enclosing query fails to resolve (`no such column`). ### File Handlers -- `FileHandler` (`file_handler.rs`) resolves `[table_name=]path` operands, - chooses the delimiter (`-F`, else comma for `.csv`, no extension and stdin, - tab otherwise) and the header policy (`.csv` and stdin: header always; - other files: headerless guess and `#` comments; `--no-header`/`--tabledef`: - no header), and routes writeback. +- `FileHandler` (`file_handler.rs`) resolves `[table_name=]path` operands + (`parse_table_spec`: the prefix names the table only when it is + identifier-like), records each file's canonical absolute path (so the same + file cannot be loaded twice, and REPL `.cd` does not change what `.save` + writes), chooses the delimiter (`-F`, else tab for `.tsv`/`.tab`, comma for + everything else and stdin) and the header policy (the first line is the + header unless `--no-header`/`--tabledef`; `#` lines are comments only + under `--comments`), and routes writeback. `load_location` loads the file + of `CREATE TABLE ... LOCATION` the same way, under the declared columns, + with the `WITH` options in place of the command-line flags. - `csv_handler.rs` does all parsing and writing. Every format is read by the `csv` crate with one configuration (RFC 4180 quoting, multi-line quoted - fields, no trimming, BOM stripped, ragged rows padded), and written back by - a small RFC 4180 record writer that also quotes fields with leading or + fields, no trimming, ragged rows padded, an unclosed quote is an error, + header names must be non-empty and unique ignoring case). Untouched rows + are written back as their source bytes; changed and inserted rows by a + small RFC 4180 record writer that also quotes fields with leading or trailing whitespace. ## File I/O Patterns @@ -205,7 +345,9 @@ This pattern provides several safety guarantees: - **Disk Full Protection**: If the disk runs out of space, the original file is preserved - **All-or-Nothing Semantics**: Either the complete new file exists, or the original file is unchanged -The temporary file is created in the same directory as the target file to ensure the rename operation is atomic (same filesystem requirement on POSIX systems). A symlinked path is resolved first, so the link's target is rewritten and the link itself is left in place. +The temporary file is created in the same directory as the target file to ensure the rename operation is atomic (same filesystem requirement on POSIX systems). A symlinked path is resolved first, so the link's target is rewritten and the link itself is left in place. The rename replaces the directory entry, so a file with several hard links is split from them: the other links keep the old contents. + +Because the rename would replace even a read-only file, `csv_handler::check_writable` runs for every target before anything is written: a file whose mode is read-only, or that cannot be opened for appending, is refused (`cannot write 'x.csv': file is read-only`), and a temp file is created and removed in its directory to prove the rename can happen. ### Delimiter Consistency @@ -221,19 +363,29 @@ This ensures that TSV files remain TSV, colon-separated files remain colon-separ Sqawk supports two primary output modes: 1. **Stdout Output**: Query results (SELECT) are written to standard output using - the `-F` delimiter, or comma when `-F` was not given. This is independent of - the delimiter a source table was parsed with -2. **File Writeback**: Modified tables (via INSERT, UPDATE, DELETE) are written back to their source files when `--write` is specified, preserving that file's delimiter + the `--output-delimiter` delimiter, else the `-F` delimiter, else comma. + This is independent of the delimiter a source table was parsed with +2. **File Writeback**: Modified tables (via INSERT, UPDATE, DELETE, ALTER TABLE, TRUNCATE) are written back to their source files when `--write` is specified, preserving that file's delimiter -For stdout output, the data is streamed directly without buffering concerns. For file writeback, the atomic write pattern described above is used. +A statement's result set is built in memory as a table and then printed. For +file writeback, the atomic write pattern described above is used. ## Storage Every table holds its rows in memory as owned values (`Vec`). Files are read whole and parsed by the `csv` crate -- RFC 4180 quoting, quoted fields -spanning lines, no trimming, UTF-8 byte order mark stripped -- and each cell's -type is inferred with a rule that round-trips (see "Type inference" below), so -an unmodified cell is written back byte for byte. +spanning lines, no trimming, UTF-8 byte order mark set aside -- and each +cell's type is inferred with a rule that round-trips (see "Type inference" +below), so an unmodified cell of a changed row keeps its text. + +A loaded table also keeps the file's bytes (`RawRows` in `table.rs`): the +whole file in one buffer plus a start/end offset per row, parallel to the +rows. That costs the file's size plus 16 bytes per row. A row keeps its +offsets while no statement changes it -- an UPDATE that sets every column to +an identical value (same type, same value) counts as no change -- and is +written back as exactly those bytes. INSERT adds rows without source bytes; +DELETE drops a row's offsets with it; ALTER TABLE ADD COLUMN and TRUNCATE +discard the source bytes of every row (every line changes). Standard input (the `-` operand) is read the same way, directly from the pipe. The resulting table has no file path, so `--write` reports it as unwritable @@ -244,26 +396,30 @@ rather than writing anywhere. - empty field: NULL - canonical integer (`42`, `-7`, where formatting the number back gives the same text): INTEGER -- canonical float (`1.5`, `-0.25`): REAL -- anything else, verbatim: TEXT -- including `007`, `1.50`, `1e3`, ` 42`, - `true`, `NaN` +- canonical float (`1.5`, `-0.25`, `2.0`, `1.0e+20`, where printing the real + the way SQLite's `CAST(x AS TEXT)` does gives the same text): REAL +- anything else, verbatim: TEXT -- including `007`, `1.50`, `1e3`, `-0.0`, + `0.30000000000000004`, ` 42`, `true`, `NaN` Text that spells a number still acts as one where a number is needed: -arithmetic, `SUM`/`AVG`, comparison against a number, and sort order. +arithmetic, `SUM`/`AVG`, every comparison (against a number or against other +numeric text), and sort order. ### Source layout Each table records how its file was laid out -- whether it had a header line, -whether records ended in CRLF, and whether `#` lines were comments -- and -writeback reproduces it. Writeback also keeps the file's permissions and, for -a symlink, rewrites the link's target and leaves the link in place. - -In a file read with comment skipping (any delimited file that is not `.csv`), -each `#` comment line is kept verbatim, with its own line ending, and -anchored to the data that follows it: comments before the header (or, in a -headerless file, before the first row) stay at the top, comments before a -row stay with that row, and comments after the last row stay at the end. So -a no-op UPDATE writes such a file back byte for byte. When rows change: +a UTF-8 byte order mark, whether records ended in CRLF, and whether `#` lines +were comments -- and writeback reproduces it. Writeback also keeps the file's +permissions and, for a symlink, rewrites the link's target and leaves the +link in place. When the file's last line had no line ending and a row is +written after it, the file's line ending is inserted first. + +The lines between records -- blank lines, and under `--comments` each `#` +comment line -- are kept verbatim, with their own line endings, and anchored +to the data that follows: lines before the header (or, in a headerless file, +before the first row) stay at the top, lines before a row stay with that +row, and lines after the last row stay at the end. So a no-op UPDATE writes +any file back byte for byte. When rows change: - UPDATE replaces rows in place, so every comment stays where it was. - DELETE leaves a deleted row's comments where they were: in front of the @@ -271,8 +427,7 @@ a no-op UPDATE writes such a file back byte for byte. When rows change: TRUNCATE) keeps all comments, in order. - INSERT appends rows after the last row and before the trailing comments. -Blank lines are not kept (the reader skips them), and `#` lines in a `.csv` -file are data, not comments. +Without `--comments`, `#` lines are data, not comments. ### Row Identity Across Updates @@ -286,12 +441,15 @@ Sqawk's in-memory database is optimized for: - **Fast Loading**: Delimiter-separated files are parsed directly into memory - **Format Flexibility**: Support for CSV, TSV, and custom-delimited files - **Efficient Filtering**: WHERE clauses are applied in a single pass -- **Low Memory Overhead**: Simple data structures minimize memory usage - **Zero Configuration**: No setup required, works directly with files in various formats -For larger datasets, consider: -- The entire dataset must fit in memory -- Complex queries may require multiple passes over the data +Memory: every table is resident for the whole run. A loaded table takes +roughly six to seven times its file's size -- each cell is a `Value` (with a +heap string for text), and the file's bytes are kept as well for verbatim +writeback. An 83 MB, one-million-row, 10-column CSV peaks at about 530 MB for +`SELECT COUNT(*)`. A statement's result set is materialized on top of that +before it prints (1.4 GB peak for `SELECT *` of the same file, 1.9 GB with +`ORDER BY`, which also holds the sorter's copy). ## Join Implementation @@ -341,15 +499,13 @@ The database engine has several architectural limitations: - **No Index Structure**: Tables are scanned in full; the only index is the temporary hash index an equi-join builds for its inner table -- **No Transaction Support**: A statement's modifications are applied once it - completes; there is no BEGIN/COMMIT and no rollback across statements +- **Transactions without savepoints**: BEGIN / COMMIT / ROLLBACK are + supported (see "Transactions" above); SAVEPOINT, RELEASE and ROLLBACK TO + are not - **Schema Flexibility**: Types are inferred per value rather than enforced - **No Constraints System**: Referential integrity not enforced - **Whole-dataset residency**: A table must fit in memory -Unsupported SQL of note: CTEs (`WITH`), parenthesized joins, and RANGE -frames with an offset, GROUPS frames and FILTER on window functions. - ---- - -*This documentation describes the current state of the Sqawk in-memory database as of the project's current version. Future versions may add additional capabilities.* \ No newline at end of file +Unsupported SQL of note: correlated derived tables and CTE bodies, +parenthesized joins, and RANGE frames with an offset, GROUPS frames and +FILTER on window functions. diff --git a/doc/sql_reference.md b/doc/sql_reference.md index 14556c2..d3d708b 100644 --- a/doc/sql_reference.md +++ b/doc/sql_reference.md @@ -2,24 +2,60 @@ ## Data Types -| Type | Aliases | -|------|---------| -| `INTEGER` | `INT` | -| `REAL` | `FLOAT`, `DOUBLE` | -| `TEXT` | `STRING` | -| `BOOLEAN` | `BOOL` | -| `NULL` | | +Every value is INTEGER, REAL, TEXT or NULL (what `TYPEOF` reports). `TRUE` +and `FALSE` are the integers 1 and 0. + +A column type declared in CREATE TABLE or ALTER TABLE ADD COLUMN is recorded +but not enforced: any type name is accepted, any value can be stored in the +column, and every value keeps its own type (see Type coercion). The name is +kept as written and `.schema` shows it, with keywords in upper case and +spacing normalized by the parser (`decimal(10, 2)` shows as `DECIMAL(10,2)`; +a name that is not a keyword, such as `money`, keeps its case). Its affinity +follows SQLite's rules, in order: a name containing `INT` is INTEGER +(`BIGINT`, `SMALLINT`); `CHAR`, `CLOB` or `TEXT` is TEXT (`VARCHAR(20)`); +`BLOB` is BLOB; `REAL`, `FLOA` or `DOUB` is REAL (`DOUBLE PRECISION`); +anything else is NUMERIC (`DECIMAL(10,2)`, `BOOLEAN`, `DATE`, `STRING`). The +affinity converts no value; it is the type `CREATE TABLE ... AS` gives a +column copied from the table. Every column needs a type: `CREATE TABLE t (a)` +is a parse error. A loaded file's columns have no declared type; `.schema` +shows them as `TEXT`. Type inference on load, per cell: an empty field is NULL; a canonical integer (`42`, `-7` -- printing the number gives back the same text) is INTEGER; a -canonical float (`1.5`, `-0.25`) is REAL; everything else is TEXT, verbatim and -untrimmed -- including `007`, `1.50`, `1e3`, ` 42`, `true` and `NaN`. Booleans -are never inferred. The rule guarantees that every loaded cell prints, and is -written back, exactly as it was read. +canonical float (`1.5`, `-0.25`, `2.0`, `1.0e+20` -- printing the real, see +Real output, gives back the same text) is REAL; everything else is TEXT, +verbatim and untrimmed -- including `007`, `1.50`, `1e3`, `-0.0`, +`0.30000000000000004` (more digits than a real prints), ` 42`, `true` and +`NaN`. Booleans are never inferred. The rule guarantees that every loaded cell +prints, and is written back, exactly as it was read. + +### Real output + +A REAL prints exactly as SQLite's `CAST(x AS TEXT)` prints it (`%!.15g`): 15 +significant digits, rounded half up from the 18- or 19-digit decimal expansion +SQLite computes in double-double arithmetic (SQLite 3.50; SQLite 3.45 does the +same when built without `long double`), so the output is identical on every +platform; positional notation when the exponent is +from -4 to 14, else exponent notation with a signed, at least two-digit +exponent; trailing zeros dropped, but a whole number keeps `.0`. So `0.1 + 0.2` +is `0.3`, `2.0 * 1` is `2.0`, `1e20` is `1.0e+20`, `1.5e-300` is `1.5e-300`, +`9223372036854775807 + 1` is `9.22337203685478e+18`, negative zero is `0.0`, +and the infinities are `Inf` and `-Inf`. The same text is what `||`, string +functions and `CAST(x AS TEXT)` see, and what `--write` writes for a changed +cell. + +Text becomes a REAL the way SQLite reads it (`sqlite3AtoF`, the same +double-double arithmetic): up to 19 significant digits scaled by powers of +ten. That is the nearest double for ordinary input, but for long or extreme +input it can be a neighbour of it -- the same neighbour SQLite reads (`1e-300` +reads as the double just below the nearest one, so `QUOTE(1e-300)` is +`1.0e-300` as in SQLite). +This applies to SQL literals, `CAST`, arithmetic on text and file loading. ## SELECT ```sql +[WITH [RECURSIVE] name [(columns)] AS (query), ...] SELECT [DISTINCT] column_list | * FROM table [alias] [, table2 ...] [JOIN ...] @@ -35,7 +71,8 @@ FROM table [alias] [, table2 ...] ```sql SELECT * -- all columns SELECT col1, col2 -- specific columns -SELECT table.col -- qualified +SELECT table.col -- qualified (by the alias, if the table has one) +SELECT main.table.col -- `main` is the one schema; any other is an error SELECT col AS alias -- aliased SELECT col alias -- alias without AS ``` @@ -49,11 +86,37 @@ SELECT col alias -- alias without AS | `AND`, `OR`, `NOT` | `a > 1 AND b < 5` | | `IS NULL`, `IS NOT NULL` | `col IS NULL` | | `LIKE`, `ILIKE` | `name LIKE 'A%'`, `code LIKE 'x\_%' ESCAPE '\'` | -| `IS [NOT] DISTINCT FROM` | `a IS NOT DISTINCT FROM b` (NULL-safe `=`) | +| `IS [NOT]`, `IS [NOT] DISTINCT FROM` | `a IS b`, `a IS NOT DISTINCT FROM b` (NULL-safe `=`) | | `BETWEEN` | `col BETWEEN 1 AND 10` | | `IN` | `col IN (1, 2, 3)` | | `IN (SELECT ...)` | `id IN (SELECT id FROM t)` | +Operators bind as in SQLite, loosest first; operators on one line share a +level and group left to right (`2 = 1 < 2` is `2 = (1 < 2)`, `2 * 3 || 4` is +`2 * (3 || 4)`, `'abc' LIKE 'A%' = 1` is `('abc' LIKE 'A%') = 1`): + +| Operators | +|-----------| +| `OR` | +| `AND` | +| `NOT` | +| `=` `==` `!=` `<>` `IS [NOT]` `IS [NOT] DISTINCT FROM` `IN` `LIKE` `BETWEEN` | +| `<` `<=` `>` `>=` | +| `&` `\|` `<<` `>>` | +| `+` `-` | +| `*` `/` `%` | +| `\|\|` | +| unary `-` `+` `~` | + +An expression may nest at most 1000 levels deep (SQLite's limit; a chain +`a + b + ...` of n terms is n levels, parentheses add none). A deeper one is +the error `expression tree is too large (maximum depth 1000)`. + +The SQL parser has its own, lower limit on *nesting*: parentheses and +function calls may nest 47 deep (`((((1))))`, `abs(abs(1))`), and scalar +subqueries inside one another about half as deep. Deeper nesting is the +error `recursion limit exceeded`. + `LIKE` follows SQLite: `%` matches any run of characters and `_` exactly one character (not byte), both including newlines; letters compare case-insensitively for ASCII only (`'a%'` matches `Alice`, but `'é'` does not @@ -85,8 +148,10 @@ only with one argument) and a filter: `COUNT(*) FILTER (WHERE age > 30)` aggregates only the rows for which the condition is true. Aggregates skip NULLs; over no rows COUNT is 0, TOTAL 0.0 and the rest NULL. SUM of integers that overflows 64 bits is an `integer overflow` error, as in SQLite (TOTAL and -AVG carry on in floating point). Text that spells a number is summed as that -number; other text is skipped. +AVG carry on in floating point). Real sums use Kahan-Babuska-Neumaier +compensated summation, as SQLite does, so `SUM` of `0.1`, `0.2`, `0.3` is +exactly `0.6`. Text that spells a number is summed as that number; other text +is skipped. ### String Functions @@ -97,22 +162,42 @@ gives NULL, and a number argument is used in its text form. |----------|-------------| | `UPPER(s)`, `LOWER(s)` | Case conversion (all of Unicode) | | `LENGTH(s)` | Number of characters | -| `TRIM(s [, chars])`, `LTRIM`, `RTRIM` | Strip whitespace, or any of `chars`, from both / the start / the end | -| `TRIM([BOTH\|LEADING\|TRAILING] chars FROM s)` | Standard-SQL spelling of the same | +| `TRIM(s)` | Strip whitespace from both ends | +| `LTRIM(s [, chars])`, `RTRIM(s [, chars])` | Strip whitespace, or any of `chars`, from the start / the end | +| `TRIM([BOTH\|LEADING\|TRAILING] chars FROM s)` | Strip any of `chars` from both ends / the start / the end | | `SUBSTR(s, start [, len])`, `SUBSTRING` | Substring (see below) | | `SUBSTRING(s FROM start [FOR len])` | Substring (alternate syntax) | | `REPLACE(s, from, to)` | Replace every occurrence; an empty `from` leaves `s` alone | | `INSTR(s, t)` | 1-based position of the first `t` in `s`, 0 if absent | | `CONCAT(a, b, ...)` | Concatenation; NULL arguments count as `''` | -| `CONCAT_WS(sep, a, ...)` | Concatenation with a separator; NULL arguments skipped | +| `CONCAT_WS(sep, a, ...)` | Concatenation with a separator; NULL arguments skipped (empty strings are not, see Differences from SQLite) | | `LEFT(s, n)`, `RIGHT(s, n)` | First / last `n` characters | | `LPAD(s, len [, fill])`, `RPAD` | Pad to `len` characters (see below) | | `HEX(x)` | Upper-case hex of the UTF-8 bytes; `''` for NULL | | `CHAR(n, ...)` | The characters with these code points | | `UNICODE(s)` | Code point of the first character | -| `PRINTF(fmt, ...)`, `FORMAT` | C-style formatting: `%d %i %u %f %e %g %x %X %o %s %c %%`, flags `-+ 0`, width, precision | +| `PRINTF(fmt, ...)`, `FORMAT` | SQLite's printf (see below) | +| `QUOTE(x)` | The SQL literal for `x`: `'it''s'`, `12`, `1.5`, `NULL` | | `a \|\| b` | Concatenation; NULL if either side is NULL | +`PRINTF` is SQLite's: conversions `%d %i %u %f %e %E %g %G %x %X %o %c %s +%z %q %Q %w %p %r %n %%`, flags `-`, `+`, space, `0`, `#` (alternate form: +`0x` / `0` prefixes, keep trailing zeros), `,` (thousands separators) and +`!` (widths and precisions in characters, and up to 26 significant digits +for reals), a width and a precision (`*` takes either from the arguments), +and the length modifiers `l` / `ll`, which are ignored. Details that follow +SQLite: `%.3d` of 7 is `007`; `%u` reads the value as unsigned 64-bit +(`-1` is `18446744073709551615`); `%q` doubles single quotes, `%Q` also +encloses the text in them (NULL is `NULL`), `%w` doubles double quotes; +`%r` is an ordinal (`1st`, `22nd`); `%.3c` repeats the character three +times; a real is printed from at most 16 significant digits, rounded half +up, so `%.20f` of 0.1 is `0.10000000000000000000`, while a precision of +zero for `%f` or `%g` rounds nothing (`%.0f` of 2.5 is `2`); `%s` widths and precisions +count bytes, not characters, unless `!` is given. A missing argument is +NULL, which prints as 0 or `''`. An unknown conversion ends the output +(`printf('a%yb')` is `a`), and a result with no output at all is NULL +(`printf('')`). + `SUBSTR` follows SQLite: positions are 1-based; a negative `start` counts from the end (`SUBSTR('hello', -3)` is `llo`); `start` 0 is a position before the first character, so `SUBSTR('hello', 0, 2)` is `h`; a negative `len` takes @@ -125,9 +210,9 @@ longer than `len` is cut to its first `len` characters (`LPAD('hello', 2)` is No function builds a string longer than SQLite's default length limit, 1,000,000,000 bytes: `LPAD('a', 10000000000)`, a `PRINTF` width or precision -past the limit, or a `REPLACE`, `HEX` or concatenation whose result would -exceed it is the error "string or blob too big". The length is checked before -anything is allocated. +past the limit (SQLite returns NULL or wraps the number), or a `REPLACE`, +`HEX` or concatenation whose result would exceed it is the error "string or +blob too big". The length is checked before anything is allocated. ### Math Functions @@ -142,16 +227,19 @@ anything is allocated. `ROUND(3.14159, 2)` is `3.14`. Halves round away from zero, judged on the value actually stored: `ROUND(2.675, 2)` is `2.67`, because 2.675 is stored as -2.67499999.... A negative `d` rounds to the left of the decimal point, so -`ROUND(1234.5, -2)` is `1200` (SQLite treats a negative `d` as 0). A `d` that -is not a number is an error rather than a silent `0`. +2.67499999..., and `ROUND(1058.05, 1)` is `1058.0`. As in SQLite, the value is +printed with `d` decimals (`%!.*f`, see Real output) and read back, and a +value beyond 2^52 is returned unchanged. `d` above 30 is 30; a negative `d` +is treated as 0, as in SQLite: +`ROUND(1234.5, -2)` is `1235.0`. A `d` that is not a number is an error rather +than a silent `0`. `ABS` of an integer is an integer, and `ABS(-9223372036854775808)` is an `integer overflow` error. `CEIL` and `FLOOR` return an integer argument unchanged. With one argument, `MIN` and `MAX` are the aggregates. -A real that lands on a whole number prints without a decimal part, as -everywhere else in sqawk: `ROUND(2.0, 2)` is `2`, where SQLite prints `2.0`. +`ROUND` returns a real, so a whole result prints with `.0` (see Real output): +`ROUND(2.0, 2)` is `2.0`. ### Arithmetic @@ -160,13 +248,28 @@ everywhere else in sqawk: `ROUND(2.0, 2)` is `2`, where SQLite prints `2.0`. - Integers stay 64-bit integers: `5 / 2` is `2`, `-7 / 2` is `-3` (truncation toward zero), `7 % 3` is `1`. Integers beyond 2^53 stay exact. - A result that does not fit in 64 bits is computed as a real instead: - `9223372036854775807 + 1` is `9223372036854775808.0`. + `9223372036854775807 + 1` is the real `9.22337203685478e+18`. - A real operand makes the result real (`7.0 / 2` is `3.5`). `%` with a real - operand takes the remainder of the operands truncated to integers, as a - real (`5.5 % 2` is `1.0`). + operand takes the remainder of the operands read as integers, as a real: + reals truncated toward zero (clamped to the 64-bit range), integers exactly, + text by its integer prefix as `CAST(text AS INTEGER)` reads it (`5.5 % 2` is + `1.0`, `9007199254740993 % 2.0` is `1.0`, `'1e3' % 7.0` is `1.0`). - Division or remainder by zero is NULL. Any NULL operand gives NULL. - Text is read by its longest numeric prefix: `'12abc' + 0` is `12`, - `' 5 ' + 1` is `6`, `'1e3' + 0` is the real `1000`, `'abc' + 1` is `1`. + `' 5 ' + 1` is `6`, `'1e3' + 0` is the real `1000.0`, `'abc' + 1` is `1`. + +The bitwise operators `&`, `|`, `<<`, `>>` and prefix `~` also follow +SQLite. They bind tighter than `<` and looser than `+` (`1 + 2 << 1` is 6). +Both operands are read as 64-bit integers: reals are truncated, text is read +by its integer prefix as `CAST(text AS INTEGER)` reads it (`'12abc' & 7` is +4). A negative shift count shifts the other way, a shift of 64 or more gives +0 (or -1 for `>>` of a negative number), `>>` keeps the sign, and a NULL +operand gives NULL. + +Integer literals may be written in hex: `0x1F`, `0XFF`. Up to 16 hex digits +are read as a 64-bit two's-complement value (`0xFFFFFFFFFFFFFFFF` is -1); +more is the error "hex literal too big". Text such as `'0x10'` is not read as +hex. ### Conditional and Conversion @@ -185,6 +288,9 @@ COALESCE, IFNULL and IIF evaluate only the argument they return. affinity rules: a name containing `INT` is INTEGER (`BIGINT`, `SMALLINT`); `CHAR`, `CLOB` or `TEXT` is TEXT (`VARCHAR(10)`); `REAL`, `FLOA` or `DOUB` is REAL; anything else (`NUMERIC`, `DECIMAL`, `BOOLEAN`, `DATE`) is NUMERIC. +One exception: `STRING` is a TEXT alias (`CAST('x' AS STRING)` is the text +`x`), where SQLite's rules make it NUMERIC. The exception is CAST's alone; a +column declared `STRING` has NUMERIC affinity (see Data Types). - `CAST(text AS INTEGER)` reads an optional sign and the digits after it: `'3.9'` is 3, `'12abc'` is 12, `' 5 '` is 5, `'1e3'` is 1, `'abc'` is 0. @@ -196,11 +302,39 @@ REAL; anything else (`NUMERIC`, `DECIMAL`, `BOOLEAN`, `DATE`) is NUMERIC. ### Result Column Names -An expression without `AS` is named after its function -- `COUNT`, `SUM`, -`ROUND(expr)`, `COALESCE(expr)` -- and a bare `CAST` or arithmetic expression -is named `expr`. `ROUND` includes its precision, so `ROUND(salary, 2)` is named -`ROUND(salary, 2)` and two roundings of one column to different precisions stay -addressable. Use `AS` whenever the header is consumed downstream. +Result columns are named as SQLite names them: + +- an `AS` alias names its column; +- a column reference, qualified or not (`u.name`, `NAME`, `(name)`), is named + by the column as the table declares it (`name`), and `*` / `t.*` expand to + the tables' column names -- in a join duplicates stay (`id,name,id,total`); +- any other expression is named by its text exactly as written, spacing and + case included: `COUNT(*)`, `n + 1`, `upper(name)||1`, + `a IS NOT DISTINCT FROM 5`; +- a compound SELECT takes its names from its leftmost SELECT. + +The columns of a derived table (`FROM (SELECT ...)`) and of a table made by +`CREATE TABLE ... AS` are named the same way and then made unique as SQLite +does: a repeated name gets `:1`, `:2`, ... (`id, id:1`). In a derived table a +column reference is named as written (`id` for a column declared `Id`). + +Use `AS` whenever the header is consumed downstream. + +### Column Positions in ORDER BY and GROUP BY + +As in SQLite, an ORDER BY or GROUP BY term that is an integer literal -- +under any parentheses and unary `+` / `-` -- whose value fits in 32 bits is +a result-column position: `ORDER BY 2`, `ORDER BY +1`, `GROUP BY (1)` and +`ORDER BY - -1` name a column. It must lie between 1 and the number of +result columns, or the statement fails with +`1st ORDER BY term out of range - should be between 1 and 2` (or +`GROUP BY`; `-1` and `0` are out of range). +Any other term is an ordinary expression, so a constant one does not reorder +rows and gives a single group: `ORDER BY 1.0`, `ORDER BY 4294967297`, +`ORDER BY 1+0`, `ORDER BY '1'`. In a compound SELECT an ORDER BY term that +is not a position must name a result column, or the statement fails with +`1st ORDER BY term does not match any column in the result set`. Inside a +window (`OVER (ORDER BY 1)`) an integer is always a constant. ### Date/Time Functions @@ -211,19 +345,37 @@ addressable. Use `AS` whenever the header is consumed downstream. | `DATETIME(t, modifier...)` | `YYYY-MM-DD HH:MM:SS` | | `JULIANDAY(t, modifier...)` | Julian day number (a real) | | `UNIXEPOCH(t, modifier...)` | Seconds since 1970-01-01 (an integer) | -| `STRFTIME(fmt, t, modifier...)` | Formatted: `%d %f %H %j %J %m %M %s %S %w %W %Y %%` | +| `STRFTIME(fmt, t, modifier...)` | Formatted (see below) | +| `TIMEDIFF(a, b)` | The time from `b` to `a`: `+YYYY-MM-DD HH:MM:SS.SSS` | | `CURRENT_DATE`, `CURRENT_TIME`, `CURRENT_TIMESTAMP`, `NOW()` | Now | -These follow SQLite, in UTC. The time string `t` is `now`, `YYYY-MM-DD`, -`YYYY-MM-DD HH:MM[:SS[.fff]]` (or with `T` for the space, and an optional -`Z` / `+HH:MM` zone), `HH:MM[:SS]` (on 2000-01-01), or a number: a Julian day -number, or seconds since 1970 when the first modifier is `unixepoch`. With no -arguments the functions return the current moment. - -Modifiers: `+N days|hours|minutes|seconds|months|years` (or `-N`; fractions -allowed), `start of day|month|year`, -`weekday N` (0 = Sunday), `unixepoch`, `utc`. `localtime` is an error, since -sqawk has no time zone. +These follow SQLite 3.45 exactly, in UTC. The time string `t` is `now`, +`[-]YYYY-MM-DD`, `YYYY-MM-DD HH:MM[:SS[.fff]]` (with any whitespace or `T` +between date and time, and an optional `Z` / `±HH:MM` zone; trailing +whitespace is allowed), `HH:MM[:SS[.fff]]` (on 2000-01-01), or a number: a +Julian day number, or seconds since 1970 with the `unixepoch` modifier. +With no arguments the functions return the current moment. Years run from +-4713 to 9999; a negative year prints with its sign (`-0001-12-31`). + +Modifiers: + +- `±N days|hours|minutes|seconds|months|years` (fractions allowed), + `±HH:MM[:SS[.fff]]`, `±YYYY-MM-DD` and `±YYYY-MM-DD HH:MM[:SS[.fff]]`; +- `start of day|month|year`, `weekday N` (0 = Sunday); +- `unixepoch`, `julianday` and `auto` (seconds since 1970 or a Julian day + number, by magnitude), each only as the first modifier of a number; +- `subsec` / `subsecond`: milliseconds in `DATETIME`, `TIME`, `UNIXEPOCH` + (then a real) and `STRFTIME('%s')`; +- `localtime` and `utc`: convert between UTC and the local time zone of the + operating system (the `TZ` environment variable where it is set). + +`STRFTIME` conversions: `%d` day, `%e` day space-padded, `%f` seconds with +milliseconds, `%F` `%Y-%m-%d`, `%H` hour, `%k` hour space-padded, `%I` / `%l` +12-hour hour, `%j` day of year, `%J` Julian day number, `%m` month, `%M` +minute, `%p` / `%P` `AM` / `am`, `%R` `%H:%M`, `%s` seconds since 1970, `%S` +seconds, `%T` `%H:%M:%S`, `%u` weekday 1-7 (Monday 1), `%w` weekday 0-6 +(Sunday 0), `%W` week of year, `%Y` year, `%%`. Any other conversion makes +the result NULL, as in SQLite 3.45. A malformed time string or an unknown modifier gives NULL. A day past the end of its month, up to 31, rolls into the next month: `DATE('2024-02-30')` is @@ -232,7 +384,7 @@ As in SQLite, a fractional month or year count moves the calendar by its whole part and then adds the fraction as 30-day months or 365-day years (`DATETIME('2024-01-01', '+1.5 months')` is `2024-02-16 00:00:00`); a count beyond a unit's range (about 14713 years) gives NULL; and the result must lie -in years 0000-9999, which is checked after the last modifier. +in years -4713 to 9999, which is checked after the last modifier. ## JOIN @@ -293,8 +445,9 @@ SELECT name FROM (SELECT name, salary FROM employees WHERE salary > 70000) t SELECT COUNT(*) FROM (SELECT department FROM employees GROUP BY department) t ``` -A derived table is materialized before the statement runs and exists only for -the statement that declares it. It may appear in any FROM -- including inside +A derived table is computed the first time the statement reads it (never, if +the code reading it never runs) and exists only for the statement that +declares it. It may appear in any FROM -- including inside a subquery, a compound arm or an INSERT ... SELECT -- and its alias is optional and may shadow a real table's name. A derived table that refers to a column of an enclosing query cannot be materialized up front and is an error. @@ -320,6 +473,100 @@ function and aggregate arguments, UPDATE SET, DELETE WHERE and INSERT VALUES. same UPDATE has already changed.) - Not supported: row values (`(a, b) IN (SELECT x, y ...)`), `ANY`/`ALL`. +## WITH (Common Table Expressions) + +```sql +WITH [RECURSIVE] name [(col, ...)] AS [[NOT] MATERIALIZED] (query) [, ...] +statement +``` + +`statement` is a SELECT (or compound), INSERT, UPDATE or DELETE. WITH may +also start the query of `INSERT INTO t WITH ... SELECT`, of `CREATE TABLE t +AS WITH ...`, of a derived table and of any subquery. + +```sql +WITH big AS (SELECT * FROM employees WHERE salary > 70000) +SELECT department, COUNT(*) FROM big GROUP BY department + +WITH totals(dept, n) AS (SELECT department, COUNT(*) FROM employees GROUP BY department) +SELECT e.name, t.n FROM employees e JOIN totals t ON t.dept = e.department + +WITH old AS (SELECT id FROM orders WHERE year < 2020) +DELETE FROM orders WHERE id IN (SELECT id FROM old) +``` + +- Each CTE is computed once, the first time the statement reads it, and + exists only for that statement. A CTE that is never read -- not + referenced, or referenced only by code that never runs, such as a + compound arm whose WHERE is false or a CASE branch not taken -- is never + computed (so its run-time errors never surface). +- A CTE is visible to the whole statement, to every CTE of its WITH list -- + those declared after it and itself included -- and to subqueries. An inner WITH + shadows an outer CTE of the same name. A CTE body resolves names where it + is defined, not where it is referenced. +- A CTE shadows a loaded table of the same name; `main.name` still names the + loaded table. The table an INSERT, UPDATE, DELETE or CREATE TABLE ... AS + writes is always a loaded table, never a CTE (`no such table: c` when only + a CTE has the name). +- The column list renames the CTE's columns and must match the body's + column count (*"table c has 1 values for 2 columns"*); without one, the + body's result columns name them, made unique like a derived table's + (`a`, `a:1`). +- A reference is qualified by its alias, else by the CTE name: + `WITH c AS (...) SELECT c.v FROM c` works, `... SELECT c.v FROM c AS x` + does not. +- `MATERIALIZED` / `NOT MATERIALIZED` are accepted and change nothing. +- Errors: *"duplicate WITH table name: c"*; *"circular reference: c"* for a + CTE that reads itself (other than as a recursive CTE below) or for CTEs + that read each other. +- A CTE body that refers to a column of an enclosing query cannot be + computed up front and is an error (`no such column`), as for a derived + table. + +### Recursive CTEs + +```sql +WITH RECURSIVE cnt(n) AS (SELECT 1 UNION ALL SELECT n + 1 FROM cnt WHERE n < 5) +SELECT n FROM cnt -- 1 2 3 4 5 + +WITH RECURSIVE sub(id, depth) AS ( + SELECT id, 0 FROM staff WHERE manager IS NULL + UNION ALL + SELECT staff.id, sub.depth + 1 FROM staff JOIN sub ON staff.manager = sub.id) +SELECT id, depth FROM sub +``` + +A CTE whose body is a `UNION` or `UNION ALL` compound with an arm that names +the CTE in its FROM is recursive, with or without the RECURSIVE keyword. As +in SQLite: + +- The arms to the left of the first self-referencing arm are the *setup*; + the trailing self-referencing arms are *recursive*. The setup's rows seed + a queue. Each row taken from the queue is output and then is the entire + content of the CTE for one run of every recursive arm (in order), whose + rows join the queue. Under `UNION` a row joins the queue only the first + time it is seen, so cycles end. +- The queue is first-in first-out (breadth-first). An `ORDER BY` on the + CTE body orders the queue instead (ties first-in first-out): `ORDER BY + depth DESC` walks a tree depth-first. Its terms are result-column + positions or the setup's result-column names. +- `LIMIT` on the CTE body stops the recursion once that many rows are + output; `OFFSET` skips output rows, which still recurse. There is no + other limit: a recursion that neither LIMIT nor a WHERE condition ends + runs until memory is exhausted, as in SQLite. +- Errors: a recursive arm may name the CTE once in its FROM (*"multiple + references to recursive table: c"*) and nowhere else (*"multiple + recursive references: c"* from a subquery); a setup arm, a derived table + or a non-UNION compound may not name it (*"circular reference: c"*); a + recursive arm may not aggregate or GROUP BY (*"recursive aggregate queries + not supported"*) or use window functions (*"cannot use window functions + in recursive queries"*); every arm has the setup's column count. + +Performance: without an ORDER BY on the body, and with no DISTINCT or `*` in +a recursive arm, each recursive arm runs once per generation of rows (the +order of the rows is still SQLite's). Otherwise it runs once per row, which +re-reads the arm's other tables for every row: slow when they are large. + ## Window Functions ```sql @@ -348,7 +595,7 @@ Supported functions: | `LAG(x [, offset [, default]])` | `x` from `offset` (default 1) rows back, else `default` (NULL) | | `LEAD(x [, offset [, default]])` | likewise, forward | | `FIRST_VALUE(x)`, `LAST_VALUE(x)`, `NTH_VALUE(x, n)` | `x` at the first / last / n-th row of the frame | -| `COUNT(*)`, `COUNT(x)`, `SUM`, `AVG`, `MIN`, `MAX` | the aggregate over the frame | +| `COUNT(*)`, `COUNT(x)`, `SUM`, `TOTAL`, `AVG`, `MIN`, `MAX`, `GROUP_CONCAT(x [, sep])`, `STRING_AGG(x, sep)` | the aggregate over the frame | Partition and order keys may be any expressions, any number of them; NULLs sort first. Rows with equal ORDER BY keys are *peers*. A query may use any @@ -378,8 +625,17 @@ integer literal), and `RANGE` with `UNBOUNDED` / `CURRENT ROW` bounds. Named windows (`WINDOW w AS (...)`, `OVER w`, `OVER (w ORDER BY ...)`) are supported. Not supported, and rejected with an error: `RANGE` frames with an offset, `GROUPS` frames, `EXCLUDE`, `FILTER` on a window function, and -`DISTINCT` inside one. SUM of integers is an integer; AVG, PERCENT_RANK and -CUME_DIST are floats. +`DISTINCT` inside one. SUM of integers is an integer; TOTAL, AVG, +PERCENT_RANK and CUME_DIST are floats. + +An aggregate over a moving frame is computed as SQLite computes it: rows enter +the frame (and leave it, when its start moves) one at a time, each once, so +any frame costs time linear in the partition. The values are SQLite's too: a +SUM that has seen a real stays a real after the real leaves the frame +(`SUM(x) OVER (ROWS 1 PRECEDING)` over 1.5, 2, 3 gives 1.5, 3.5, 5.0), reals +that leave are subtracted from the running compensated sum, GROUP_CONCAT +drops a leaving value and the separator after it, and an empty GROUP_CONCAT +is NULL. ## Set Operations @@ -390,6 +646,11 @@ SELECT ... INTERSECT SELECT ... -- rows in both SELECT ... EXCEPT SELECT ... -- rows in first but not second ``` +As in SQLite, the four operators have equal precedence and apply left to +right: `A UNION ALL B INTERSECT C` is `(A UNION ALL B) INTERSECT C`. +Parenthesize an operand to group differently. A compound may have at most +500 terms (`too many terms in compound SELECT`). + ## INSERT ```sql @@ -436,16 +697,64 @@ CREATE TABLE name ( ) [LOCATION 'path'] [STORED AS TEXTFILE] -[WITH (DELIMITER='...')] +[WITH (DELIMITER='...', HEADER=true|false, COMMENTS=true|false)] ``` ```sql CREATE TABLE name AS SELECT ... FROM ... ``` -The new table's columns are the query's result columns. Column and table -constraints (`PRIMARY KEY`, `NOT NULL`, `DEFAULT`, ...) are rejected: sqawk -does not enforce them. +Column types are recorded, not enforced (see Data Types). With `LOCATION`, +the declared columns are the schema of that file, as for a Hive external +table: + +- **The file exists**: it is loaded under the declared column names, by the + same loader as an input file. Its first line is a header: it is skipped + as data, must have exactly one field per declared column (else + `x.csv: line 1: header has 3 fields, but the table declares 2 columns`), + and is written back as it was read -- the declared names do not replace + its text. A record with more fields than columns is an error; a short one + is padded with NULL. Loading is not a modification: with no INSERT, + UPDATE, DELETE, ALTER TABLE or TRUNCATE, `--write` leaves the file alone, + and rows no statement changed are written back byte for byte. +- **The file does not exist**: the table starts empty and `--write` creates + the file, even if nothing is inserted (the file then holds only the + header line of the declared names). + +`WITH` options: + +- `DELIMITER='c'`: the file's delimiter. Without it, tab for a `.tsv` or + `.tab` path and comma for any other -- the rule an input file is loaded + with. +- `HEADER=true|false` (default `true`): with `false` the file's first line + is data and the file is written with no header line. +- `COMMENTS=true|false` (default `false`): with `true`, lines starting with + `#` are comments, kept in place on writeback (the `--comments` rule). + +`HEADER` and `COMMENTS` take `true` or `false` in any case, quoted or not, +and need `LOCATION`. A relative path is resolved against the current +directory when the statement runs. A file already loaded -- as an input file +or by another `CREATE TABLE ... LOCATION` -- is an error (`file x.csv is +already loaded as table x`), as is a directory or a file that cannot be read. +Inside a transaction, `ROLLBACK` removes the table and never touches the +file. + +```sql +-- people.csv holds "num,label\n1,alice\n2,bob\n" +CREATE TABLE p (id INT, name TEXT) LOCATION 'people.csv'; +SELECT name FROM p WHERE id = 2; -- bob + +-- pairs.txt holds "1;alice\n2;bob\n" (no header line) +CREATE TABLE q (id INT, name TEXT) LOCATION 'pairs.txt' + WITH (DELIMITER=';', HEADER=false); +``` + +`CREATE TABLE ... AS` gives the new table the query's result columns. +Column and table constraints (`PRIMARY KEY`, `NOT NULL`, `DEFAULT`, ...) are +rejected: sqawk does not enforce them. `STORED AS` takes only `TEXTFILE` and `WITH` only +`DELIMITER`, `HEADER` and `COMMENTS`; anything else there is an error. A table without `LOCATION` +(every `CREATE TABLE ... AS` table among them) has no file and lasts for the +run; `CREATE TEMP TABLE` is accepted for such a table. ## DROP TABLE @@ -488,27 +797,78 @@ NULL and the empty string both print as an empty field. Values are typed per cell as they are read, so one column may hold integers, floats and text. -When a comparison mixes a number and a string, the string is converted to a -number if it parses as one, and the two are compared textually otherwise. This -matches arithmetic, so `x + '1'` and `x > '1'` agree — awk-like rather than -strict SQL, which suits untyped delimited input. Two strings compare as text. - -Text that spells a number (`007`, `1.50`, `1e3`, which load as text to keep -their spelling) is used as that number by arithmetic and by `SUM` and `AVG`. -Arithmetic reads any text by its numeric prefix (see Arithmetic). In a -condition (WHERE, `NOT`, `AND`, CASE), text is likewise read as a number, so +Every comparison -- `=`, `<>`, `<`, `<=`, `>`, `>=`, `IS`, `BETWEEN`, `IN` +with a list, `IN (SELECT ...)`, `CASE x WHEN`, `NULLIF`, JOIN conditions +(including hash joins), `ORDER BY`, `MIN` / `MAX` and window ordering -- uses +one rule: + +- A value's *numeric reading* is the value itself for an integer or real, and + for text the number it spells in full: an optional sign, digits with an + optional point, an optional exponent, optionally surrounded by ASCII + whitespace (`'007'`, `' 1.5 '`, `'1e3'`, `'+3'`). `'12abc'`, `'0x10'`, + `'inf'`, `'NaN'` and `''` have none. +- Two values that both have a numeric reading compare as numbers, whatever + their types -- text against text included: `'10' > '9'`, `'1' = '1.0'`, + `'007' = 7`, and integers compare exactly (no rounding through floating + point). +- A number (or numeric-reading text) is less than text without one: + `10 < 'abc'`, `'-x' > 10`. +- Two texts without a numeric reading compare byte by byte (BINARY): + `'B' < 'a'`. +- Any NULL operand makes the comparison UNKNOWN (sorting puts NULLs first). + +`<` and `ORDER BY` therefore agree: a column sorts `9, 10, -x, abc`, and +`WHERE s < '9'` selects nothing from it. `x IN (SELECT ...)` matches exactly the +rows `x = ...` (and the equivalent JOIN) matches: `'9' IN (SELECT 9)` is 1. + +Values equal under this rule but not the same value -- text `'1'` and integer +`1`, or text `'007'` and `'7'` -- stay apart in `DISTINCT`, `GROUP BY`, +`UNION` / `INTERSECT` / `EXCEPT` and `COUNT(DISTINCT ...)`, as in SQLite; a +number sorts before text of equal value, and equal texts by spelling. An +integer and a real of equal value (`1` and `1.0`) are one value. + +Text that spells a number is used as that number by arithmetic and by `SUM` +and `AVG`. Arithmetic reads any text by its numeric prefix (see Arithmetic). In +a condition (WHERE, `NOT`, `AND`, CASE), text is likewise read as a number, so `'x'` is false and `'1x'` true, as in SQLite. -`ORDER BY`, `GROUP BY`, `MIN` and `MAX` order such text among the numbers by -value; values equal as numbers but spelled differently (`7` and `007`) remain -distinct groups. ## Writeback -Modifications (INSERT, UPDATE, DELETE) remain in-memory unless `--write` flag is specified. -Cells a statement did not change are written back byte for byte; NULL is -written as an empty field; the source file's delimiter, header line (or its -absence), line endings, `#` comment lines (in a non-`.csv` file) and -permissions are kept. +Modifications remain in memory unless the `--write` flag is specified. Then +every table that INSERT, UPDATE, DELETE, ALTER TABLE or TRUNCATE changed, or +that `CREATE TABLE ... LOCATION` created over a file that did not exist, is +written to its file. A table +with no file (CREATE TABLE without LOCATION, CREATE TABLE ... AS) is +skipped, and a table dropped by DROP TABLE is not written. +Rows no statement changed are written back byte for byte, exactly as they +were read; a changed or inserted row is written from its values, NULL as an +empty field. The source file's delimiter, header line (or its absence), byte +order mark, line endings, blank lines, `#` comment lines (under +`--comments`) and permissions are kept. + +Every target file is checked before any is written: a read-only file fails +the run (`cannot write 'x.csv': file is read-only`) and no file is written. +Each file is replaced atomically by renaming a temp file over it, so a file +with several hard links is split from them (the other links keep the old +contents); a symlink is followed and stays in place. + +## Transactions + +```sql +BEGIN [DEFERRED | IMMEDIATE | EXCLUSIVE] [TRANSACTION] +COMMIT [TRANSACTION] +END [TRANSACTION] +ROLLBACK [TRANSACTION] +``` + +`ROLLBACK` restores every table (rows, schema, created and dropped tables) to +its state at `BEGIN`; `COMMIT` / `END` keeps the changes. The three `BEGIN` +modifiers are accepted and mean plain `BEGIN`. As in SQLite, `BEGIN` inside a +transaction is an error (`cannot start a transaction within a transaction`), +as are `COMMIT` and `ROLLBACK` outside one (`cannot commit - no transaction +is active`). A transaction still open after the last statement, or when the +REPL exits, is rolled back, before any `--write` writeback. `SAVEPOINT`, +`RELEASE` and `ROLLBACK TO` are not supported. ## Unsupported syntax @@ -520,7 +880,9 @@ nothing is modified. That includes: and `RETURNING` on INSERT, UPDATE and DELETE (`REPLACE INTO` does not parse) - `UPDATE ... FROM`, `DELETE ... USING`, `DELETE t FROM t JOIN ...`, and UPDATE / DELETE with `ORDER BY` or `LIMIT` -- `WITH` (common table expressions), `FETCH FIRST`, `FOR UPDATE`, `TOP`, +- `WITH` on a parenthesized compound operand (`SELECT 1 UNION (WITH ...)`), + a CTE body that is an INSERT / UPDATE / DELETE, typed CTE columns +- `FETCH FIRST`, `FOR UPDATE`, `TOP`, `QUALIFY`, `SORT BY` / `CLUSTER BY` / `DISTRIBUTE BY`, `LATERAL VIEW`, `SELECT ... INTO`, `GROUP BY ... WITH ROLLUP` - `TABLESAMPLE`, table hints, `LATERAL` subqueries, column aliases on a FROM @@ -529,11 +891,20 @@ nothing is modified. That includes: list, `FILTER` or `DISTINCT` on a scalar function, `FILTER` on a window function - `CAST ... FORMAT`, `LIKE ANY`, blob literals (`X'00'`) and parameters +- `COLLATE` (any collation, in expressions and `ORDER BY`) +- a RIGHT JOIN whose left side already contains an outer join, a FULL JOIN + followed by further joins, parenthesized joins, and `INTERSECT ALL` / + `EXCEPT ALL` - table and column constraints, `CREATE TABLE IF NOT EXISTS`, - `PARTITIONED BY`, `ROW FORMAT` + `PARTITIONED BY`, `ROW FORMAT`, `STORED AS` any format but `TEXTFILE`, + `WITH` options other than `DELIMITER` / `HEADER` / `COMMENTS` (and + `HEADER` / `COMMENTS` without `LOCATION`), table comments, and `LOCATION` / + `STORED AS` / `WITH` or a column list on `CREATE TABLE ... AS` +- savepoints, transaction modes (`ISOLATION LEVEL`, `READ ONLY`) and + `START TRANSACTION` - statements other than SELECT, INSERT, UPDATE, DELETE, CREATE TABLE, DROP - TABLE, ALTER TABLE ADD COLUMN and TRUNCATE (no views, indexes, triggers or - transactions) + TABLE, ALTER TABLE ADD COLUMN, TRUNCATE and BEGIN / COMMIT / ROLLBACK (no + views, indexes or triggers) ## Differences from SQLite @@ -541,18 +912,22 @@ sqawk aims to give SQLite's answers. Where it deliberately does not: | Area | SQLite | sqawk | |------|--------|-------| -| Column types | Declared affinity converts `'10'` stored in an INTEGER column to 10 | Each cell is typed when read; a text/number comparison converts the text if it is a number (`a = '10'` matches 10) -- awk-like, see Type coercion | -| Real output | `%!.15g`: `1.0`, `9.22337203685478e+18` | Shortest round-trip digits, no exponent, no `.0`: `1`, `9223372036854775808` | +| Column types | Declared affinity converts `'10'` stored in an INTEGER column to 10; text compares with text as text (`'10' < '9'`) | Each cell is typed when read; numeric-looking text compares numerically everywhere, text against text included (`a = '10'` matches 10, `'10' > '9'`), see Type coercion | | NULL vs `''` | Distinct in output | Both print as an empty field | +| `CAST(x AS STRING)` | NUMERIC affinity: `CAST('x' AS STRING)` is the integer `0` | TEXT: `CAST('x' AS STRING)` is the text `x` | | `UPPER` / `LOWER` | ASCII letters only | All of Unicode | | `TRIM(s)` | Strips spaces only | Strips all whitespace | -| `ROUND(x, -d)` | Negative `d` treated as 0 | Rounds to the left of the point | -| `SUM` over text | Non-numeric text counts as 0 (result real) | Non-numeric text is skipped | +| `CONCAT_WS` with an empty-string argument | SQLite 3.45 skips it like NULL: `CONCAT_WS(',', 'a', '', 'b')` is `a,b` | The separator is written: `a,,b` (as in SQLite 3.50 and PostgreSQL); only NULL arguments are skipped | +| `SUM` / `AVG` over text | Non-numeric text counts as 0: `SUM` becomes real, `AVG` divides by every non-NULL value, and over only non-numeric text both give `0.0` | Non-numeric text is skipped: `SUM` stays an integer over integers and integer text, `AVG` divides by the count of numeric values, and over only non-numeric text both give NULL | +| Bare column in an aggregate query (`SELECT k, MAX(y), s FROM t GROUP BY k`, `SELECT COUNT(*), x FROM t`) | Allowed: the value comes from one row of the group | Rejected: `'s' must appear in GROUP BY or be used in an aggregate` | | `INSERT OR REPLACE`, `ON CONFLICT` | Plain INSERT when the table has no unique constraint | Rejected: sqawk tables have no keys, so the statement cannot do what it asks | | Subqueries in UPDATE / DELETE | A correlated subquery sees rows already changed | Sees the table as it was before the statement | +| Aggregate of outer-query columns in a subquery (`(SELECT MIN(t.x) FROM u)`) | Evaluated in the outer query, which becomes an aggregate query | Rejected; an aggregate that also reads a column of the subquery's own tables aggregates in the subquery, as in SQLite | +| Derived table (`FROM (SELECT ...)`) reading an outer-query column | Correlated | Rejected (`no such column`): a derived table is computed once, before the statement | +| CTE body reading an outer-query column (`SELECT (WITH c AS (SELECT t.x) SELECT * FROM c) FROM t`) | Correlated | Rejected (`no such column`): a CTE is computed once, on first reference | | `ILIKE`, `LPAD`, `RPAD`, `LEFT`, `RIGHT`, `TRIM(... FROM ...)` | Not available | Available | -| `localtime` date modifier | Local time zone | Error (UTC only) | -| Parser | SQLite grammar | sqlparser's Hive dialect: `GLOB`, `REGEXP`, `x IS 10`, `LIMIT n, m`, `TRIM(s, chars)`, `REPLACE INTO` do not parse | +| `PRINTF` width or precision past the length limit | NULL, or the number wrapped to 31 bits | Error "string or blob too big" | +| Parser | SQLite grammar | sqlparser's Hive dialect with SQLite's operator precedence: `GLOB`, `LIMIT n, m`, `TRIM(s, chars)`, `REPLACE INTO`, `x IN ()` do not parse, `REGEXP` is rejected, and parentheses nest at most 47 deep | -Not implemented: common table expressions, views, indexes, triggers, +Not implemented: views, indexes, triggers, `RETURNING`, row values, `GLOB`, and blob values. diff --git a/doc/user_guide.md b/doc/user_guide.md index c68b229..b49f0d2 100644 --- a/doc/user_guide.md +++ b/doc/user_guide.md @@ -12,29 +12,37 @@ - [Your First Sqawk Command](#your-first-sqawk-command) - [How Sqawk Processes Files](#how-sqawk-processes-files) 4. [Command Line Options](#command-line-options) + - [Options at a Glance](#options-at-a-glance) - [SQL Statement Option (-s)](#sql-statement-option--s) + - [SQL File Option (-f)](#sql-file-option--f) - [Interactive Mode (-i)](#interactive-mode--i) - [Write Flag (--write)](#write-flag---write) - [Field Separator Option (-F)](#field-separator-option--f) + - [Output Options](#output-options) - [Table Definition Option (--tabledef)](#table-definition-option---tabledef) + - [Header Rows](#header-rows) - [Verbose Mode (-v)](#verbose-mode--v) + - [Errors and Exit Status](#errors-and-exit-status) - [Help (--help)](#help---help) 5. [Working with Files](#working-with-files) - [File Format Support](#file-format-support) - - [Reading Standard Input](#reading-standard-input) + - [Defining Schemas with CREATE TABLE](#defining-schemas-with-create-table) - [Table Naming](#table-naming) - [Handling Multiple Files](#handling-multiple-files) - [File Writeback Behavior](#file-writeback-behavior) - - [Defining Schemas with CREATE TABLE](#defining-schemas-with-create-table) + - [Transactions](#transactions) 6. [Examples](#examples) - [Basic Queries](#basic-queries) - [Aggregation and Grouping](#aggregation-and-grouping) - [Distinct Values](#distinct-values) - [Joins](#joins) + - [Common Table Expressions (WITH)](#common-table-expressions-with) - [Data Modification](#data-modification) - [String Functions](#string-functions) + - [SQL Semantics](#sql-semantics) - [System Files](#system-files---tabledef) - [Output Redirection](#output-redirection) + - [Reading Standard Input](#reading-standard-input) - [Interactive REPL Session](#interactive-repl-session) 7. [Working with Large Files](#working-with-large-files) 8. [Troubleshooting](#troubleshooting) @@ -54,6 +62,7 @@ Sqawk is an SQL-based command-line tool for processing delimiter-separated files - Automatic type inference and cross-file operations - Powerful SQL dialect including joins, sorting, filtering, and aggregations - Subqueries (scalar, `IN`, `EXISTS`, correlated) and derived tables +- Common table expressions (`WITH`), including recursive ones - Set operations (UNION, INTERSECT, EXCEPT) and window functions - Interactive REPL mode for SQL exploration and execution - Safe operation with explicit write-back control @@ -67,10 +76,11 @@ Sqawk is designed for data analysts, developers, system administrators, and anyo The simplest way to install Sqawk is through Cargo, Rust's package manager: ```sh -cargo install sqawk +cargo install --locked sqawk ``` -This will download, compile, and install the latest version of Sqawk from crates.io. +This downloads, compiles and installs the latest version of Sqawk from +crates.io; `--locked` builds with the dependency versions it was tested with. ### Building from Source @@ -84,8 +94,7 @@ To build from source: 2. Build and install using Cargo: ```sh - cargo build --release - cargo install --path . + cargo install --locked --path . ``` ### Installation Notes @@ -95,9 +104,10 @@ To build from source: - The binaries are self-contained with no runtime dependencies - Installing the crate provides the `sqawk` command. The source tree also contains `tsq`, a development-only generator of deterministic multi-table - CSV data and SQL queries for exercising sqawk; it is not installed. Run + test data, differential tester against SQLite, writeback and command-line + fuzzer and performance runner; it is not installed. Run `cargo run -p tsq -- --help` from a checkout, or see the - [project README](../README.md#tsq---test-data-generator-development-only) + [project README](../README.md#tsq---test-data-generator-differential-tester-and-fuzzer-development-only) ## Getting Started @@ -113,7 +123,12 @@ Where: - `OPTIONS` include SQL statements and other flags - `FILES` are the delimiter-separated files to process. At least one is required. Each may be written as `[table_name=]file_path`, and a path of `-` - reads [standard input](#reading-standard-input) + reads [standard input](#reading-standard-input). The part before `=` names + the table only when it is made of letters, digits and `_` (so a path such + as `k=v/e.csv` in a directory named `k=v` still works when `v/e.csv` does + not exist); otherwise the table is named after the file without its + extension. An empty table name (`=e.csv`, or a file named `.csv`) is an + error, and so is loading the same file twice under different names ### Your First Sqawk Command @@ -146,8 +161,8 @@ When you run a Sqawk command: 1. Sqawk loads each specified file into memory as a table 2. Table names are derived from file names (without extensions) or can be explicitly assigned -3. The first row is treated as column headers (always for `.csv` files and - standard input; other files are checked for a missing header -- see +3. The first line of every file is its header row of column names, unless + `--no-header` or `--tabledef` says the file has none (see [Header Rows](#header-rows)) 4. Data types are inferred per cell: an empty field is NULL, a number written in its canonical form (`42`, `1.5`) is a number, and everything else is @@ -158,6 +173,25 @@ When you run a Sqawk command: ## Command Line Options +### Options at a Glance + +| Option | Description | +|--------|-------------| +| `-s SQL`, `--sql SQL` | SQL statement(s) to run; repeatable | +| `-f FILE`, `--file FILE` | Run the SQL statements in FILE (`-` reads standard input); repeatable, runs in order with `-s` | +| `-i`, `--interactive` | Start the interactive REPL | +| `-w`, `--write` | Write modified tables back to their files | +| `-F SEP`, `--field-separator SEP` | Input field separator (one character, or `\t`); also the output delimiter unless `--output-delimiter` is given | +| `--output-delimiter SEP` | Delimiter for query results (default: the `-F` separator, else comma) | +| `--no-output-header` | Omit the header line of every result set | +| `--no-header` | Input files have no header row | +| `--comments` | Skip lines starting with `#` (kept on `--write`) | +| `--tabledef NAME:COL,...` | Name the columns of a headerless table | +| `-v`, `--verbose` | Debugging output on stderr; its format is not stable | +| `-h`, `--help` / `-V`, `--version` | Help / version | + +At least one `-s` or `-f` is required unless `-i` is given. + ### SQL Statement Option (-s) The `-s` option specifies an SQL statement to execute: @@ -174,6 +208,30 @@ sqawk -s "SELECT COUNT(*) FROM data" -s "SELECT AVG(value) FROM data" data.csv Statements are executed in sequence, with each operating on the current state of the tables. +### SQL File Option (-f) + +`-f FILE` (or `--file FILE`) runs the SQL statements in a file. A file may +hold several statements separated by `;`, and a file with no statements +(empty, or only whitespace) runs nothing: + +```sh +sqawk -f report.sql employees.csv + +# Read the statements from standard input +echo "SELECT COUNT(*) FROM employees" | sqawk -f - employees.csv +``` + +`-s` and `-f` may be mixed and repeated; they run in the order they appear +on the command line, so here the UPDATE in `fix.sql` runs between the two +SELECTs: + +```sh +sqawk -s "SELECT COUNT(*) FROM data" -f fix.sql -s "SELECT COUNT(*) FROM data" data.csv +``` + +`-f -` and a `-` file operand cannot be used together: standard input can +supply the SQL or a table, not both. + ### Interactive Mode (-i) The `-i` (or `--interactive`) option launches Sqawk in REPL (Read-Eval-Print Loop) mode, providing an interactive SQL shell similar to the sqlite3 command-line utility: @@ -188,13 +246,20 @@ This opens an interactive SQL prompt where you can: - Execute multiple statements in sequence - Toggle settings like write mode +`-i` cannot be combined with `-s` or `-f`; that is a usage error (exit +status 2). + +A statement runs once its text ends with `;` (outside quotes and comments); +until then Enter continues it on the next line. A dot-command is always a +single line. + **REPL Commands:** | Command | Description | |---------|-------------| | `.cd DIRECTORY` | Change the working directory | | `.changes [on\|off]` | Show the number of rows changed by each statement | -| `.exit [CODE]` | Exit the REPL, optionally with an exit code | +| `.exit [CODE]` | Exit the REPL, with exit status `CODE` when given | | `.help` | List the available commands | | `.load [TABLE=]FILE` | Load `FILE`, optionally under the name `TABLE` | | `.print STRING...` | Print a literal string | @@ -203,7 +268,7 @@ This opens an interactive SQL prompt where you can: | `.schema [TABLE]` | Show the schema for `TABLE`, or for every table | | `.show [WHAT]` | Show current settings and status | | `.stats [on\|off]` | Toggle statistics display | -| `.tables [PATTERN]` | List tables, optionally matching a LIKE pattern | +| `.tables [PATTERN]` | List tables, optionally only those matching a LIKE pattern (`%`, `_`; case-insensitive) | | `.version` | Show source, library and compiler versions | | `.write [on\|off]` | Toggle writing changes to files (default off) | @@ -220,12 +285,10 @@ sqawk> SELECT name, department FROM employees WHERE salary > 70000; name,department Alice,Engineering Charlie,Engineering - sqawk> .tables Tables: employees sales - sqawk> .schema employees CREATE TABLE employees ( id TEXT, @@ -233,27 +296,74 @@ CREATE TABLE employees ( department TEXT, salary TEXT ); - sqawk> .changes on Changes display enabled - sqawk> UPDATE employees SET salary = salary * 1.1 WHERE department = 'Engineering'; 2 rows affected Changes not saved: use .write to save changes to files - +sqawk> SELECT nmae FROM employees; +sqawk: no such column: nmae sqawk> .save employees Changes saved to table 'employees' - sqawk> .exit ``` -Two things in that transcript are worth calling out: +Things in that transcript worth calling out: -- `.schema` reports every column as `TEXT`. Loaded files carry no declared - schema -- types are inferred per value at comparison time, not per column, so - there is no column type for `.schema` to report. See +- `.schema` reports every column of a loaded file as `TEXT`. Loaded files + carry no declared schema -- types are inferred per value at comparison + time, not per column, so there is no column type for `.schema` to report. + A table made with CREATE TABLE shows each column's type as declared + (`BIGINT`, `VARCHAR(20)`, ...). See [Type coercion](sql_reference.md#type-coercion). - Row counts are only printed after `.changes on`; the default is off. +- Query results print exactly as with `-s`: the header line, then the rows. +- An error is printed as `sqawk: ` on standard error and the + session continues. + +**Line editing, completion and history.** When standard input is a +terminal, the REPL is a line editor. Tab completes dot-commands, SQL +keywords, function names, table names and column names (case-insensitive +prefix match); after `table.` it completes that table's columns, and after +`.schema`, `.save` or `.tables` it completes table names. Ctrl-C abandons +the line being typed; Ctrl-D (end of input) leaves the REPL. + +History is kept only when standard input is a terminal, in +`$XDG_STATE_HOME/sqawk/history` when `XDG_STATE_HOME` is an absolute path +(the directory is created as needed). Otherwise -- `XDG_STATE_HOME` unset, +empty or relative -- it is `$HOME/.sqawk_history` when `HOME` is set and +non-empty, and when neither applies no history is kept. + +**Piping a script.** When standard input is not a terminal, `sqawk -i` +reads commands from it line by line and prints no banner, no prompts and no +status messages (`Changes display enabled`, `Changes not saved ...`, +`Loaded table ...`: with `-v` they go to standard error). Standard output +carries only what the commands produce -- query results, `.changes` row +counts, and the output of `.tables`, `.schema`, `.print`, `.show`, `.help` +and `.version`: + +``` +$ printf 'SELECT name FROM employees LIMIT 2;\n.tables\nSELECT COUNT(*) FROM employees' | sqawk -i employees.csv +name +Alice +Bob +Tables: + employees +COUNT(*) +8 +``` + +A final statement with no `;` runs at end of input. Like sqlite3, an error +does not stop the script: it is reported, the remaining commands run, and +the exit status is 1. + +**Exit status.** + +- `.exit CODE` exits with status `CODE` (taken modulo 256), in a terminal + session or a script, whatever errors came before +- otherwise a piped script exits with status 1 if any SQL statement or + dot-command failed, else 0 +- a terminal session exits with status 0 (errors there do not change it) The interactive mode is particularly useful for: - Exploring datasets without writing multiple commands @@ -270,45 +380,82 @@ sqawk -s "DELETE FROM data WHERE status = 'expired'" data.csv --write ``` Important notes about the write behavior: -- Only tables that were actually modified by an operation (INSERT, UPDATE, DELETE) are saved -- Cells the statement did not change are written back exactly as they were - read: `007` stays `007`, `1.50` stays `1.50`, quoted text keeps its quotes - where they are needed. The one normalisation is that quotes a field did - not need (`"plain"`) are dropped -- The delimiter, the header line (or its absence), CRLF line endings, the - file's permissions and row order are preserved. Writing through a symlink - updates the file it points to and leaves the link in place +- Only tables that were actually modified are saved: by INSERT, UPDATE, + DELETE, ALTER TABLE or TRUNCATE, or created by `CREATE TABLE ... + LOCATION 'file'` over a file that did not exist (an existing `LOCATION` + file is written only if a statement changes its table). A table created + without `LOCATION` (including `CREATE + TABLE ... AS SELECT`) is a scratch table: it has no file and is skipped. A + table that was modified and then dropped is not written +- Every file is checked before any is written: if one is read-only (or its + directory cannot be written), sqawk reports `cannot write 'x.csv': file is + read-only`, exits with status 1 and writes no file at all +- A transaction (`BEGIN`) still open after the last statement is rolled back + first, so uncommitted changes are never written +- Rows the statements did not change are written back byte-for-byte as they + were read: quoting (even quotes a field did not need), unquoted spaces, + short (ragged) rows, line endings, invalid UTF-8 -- all kept. A row that + was changed or inserted is written from its values, with quotes only where + a field needs them; its unchanged cells keep their text (`007` stays `007`, + `1.50` stays `1.50`) +- The delimiter, the header line (or its absence), a UTF-8 byte order mark, + blank lines, `--comments` comment lines, the file's permissions and row + order are preserved. A row appended after a last line that had no line + ending starts on a new line. Writing through a symlink updates the file it + points to and leaves the link in place +- The file is replaced atomically (a temp file renamed over it), so a file + with several hard links is split from them: the file named on the command + line gets the new contents, and the other links keep the old ones - NULL is written as an empty field - Without `--write`, your files remain untouched regardless of the SQL operations ### Field Separator Option (-F) -The `-F` option allows you to specify a custom field separator for your files: +The `-F` option (long form `--field-separator`) allows you to specify a custom field separator for your files: ```sh -# Process a tab-delimited file -sqawk -F '\t' -s "SELECT * FROM data" data.tsv +# Process a tab-delimited file that does not end in .tsv +sqawk -F '\t' -s "SELECT * FROM data" data.txt # Process a pipe-delimited file sqawk -F '|' -s "SELECT * FROM logs" logs.txt ``` Notes on field separators: -- Files ending in `.csv` default to comma; **any other extension defaults to - tab**, `.tsv` and `.txt` alike. A comma-separated `.txt` file therefore needs - an explicit `-F,`. A file with no extension at all defaults to comma +- Files ending in `.tsv` or `.tab` default to tab; every other file (any + other extension, or none) and standard input default to comma - Common separators include tab (`\t`), comma (`,`), colon (`:`), and pipe (`|`) - The separator must be a single ASCII character other than a quote or line break; the two-character escape `\t` means tab. Anything else (including an - empty `-F ''`) is rejected with an error + empty `-F ''`) is a usage error (exit status 2) - `-F` applies to every input file in the invocation, not per file - The input separator is preserved when writing a table back with `--write` -- Query results on stdout use `-F` when it is given, and comma otherwise -- - reading a tab-separated file without `-F` prints comma-separated results +- Query results on stdout use `--output-delimiter` when it is given, else + `-F`, else comma -- reading a tab-separated file without either prints + comma-separated results (see [Output Options](#output-options)) + +### Output Options + +Query results are written to stdout as a delimited file, one result set per +statement, each starting with a header line of column names. + +- `--output-delimiter SEP` sets the delimiter between result fields. It + follows the same rules as `-F` (one ASCII character other than a quote or + line break, or `\t`). Without it, results use the `-F` separator when one + is given, and comma otherwise +- `--no-output-header` omits the header line of every result set + +Both apply in the REPL too. + +```sh +# Read a pipe-separated file, print tab-separated results without a header +sqawk -F '|' --output-delimiter '\t' --no-output-header \ + -s "SELECT name, total FROM sales" sales.txt +``` ### Table Definition Option (--tabledef) -The `--tabledef` option allows you to define column names for files that don't have header rows, such as system files like `/etc/passwd`: +The `--tabledef` option allows you to define column names for files that don't have header rows, such as system files like `/etc/passwd`. A table named by `--tabledef` has no header row: its first line is data. ```sh # Process /etc/passwd with meaningful column names @@ -319,8 +466,14 @@ sqawk -F: --tabledef=passwd:username,password,uid,gid,gecos,home,shell \ Format: `--tabledef=table_name:col1,col2,col3,...` +A definition without `:`, with an empty table or column name, naming a +column twice, or defining the same table twice is an error, and so is a +definition for a table that no input file loads. + This is useful for: -- System files like `/etc/passwd`, `/etc/group`, `/etc/hosts` +- System files like `/etc/passwd` and `/etc/group` (files separated by runs + of whitespace, such as `/etc/hosts`, are not delimited files and are not + supported) - Log files with fixed column formats - Any file without a header row @@ -336,38 +489,56 @@ sqawk -F: \ ### Header Rows -- A `.csv` file's first line is always its header, as is the first line of - standard input. Lines starting with `#` are data -- Any other file (`.tsv`, `.txt`, no extension) is checked: if its first line - looks like data rather than column names -- a number, a path starting with - `/`, `*`, `root` or `nobody` -- it is treated as data and the columns are - named `a`, `b`, `c`, ... Lines starting with `#` are comments and skipped +- The first line of every file, whatever its extension, and of standard + input is its header. sqawk never guesses - `--no-header` says no input file has a header row: the first line is data and the columns are named `a`, `b`, `c`, ... (or by `--tabledef`) - `--tabledef` likewise means the file has no header row +- Column names are case-insensitive, so a header must not repeat a name + (`id,ID` is an error) or leave one empty (`a,,c` is an error) A table loaded without a header row is written back without one. ```sh sqawk --no-header -s "SELECT a, b FROM pairs WHERE b > 10" pairs.csv + +# /etc/passwd: colon-separated, no header row +sqawk -F: --no-header -s "SELECT a, f FROM passwd WHERE c >= 1000" /etc/passwd ``` ### Verbose Mode (-v) -The verbose mode provides additional information about the operations being performed: +`-v` (or `--verbose`) prints debugging output about what sqawk is doing: ```sh sqawk -s "SELECT * FROM data WHERE id > 1000" data.csv -v ``` Verbose output goes to standard error, so it never mixes with query results. -It includes: -- SQL statements being executed -- Number of rows affected or returned -- Table loading information -- Write status (whether changes were saved) +It includes the tables loaded, each SQL statement, its compiled bytecode and +every executed instruction, row counts, and whether changes were saved. +Expect a lot of it on large inputs. + +The output is for debugging; its format is not stable, so do not parse it. +A closed or failing stderr never stops a run. -This mode is particularly useful for debugging or understanding exactly what Sqawk is doing with your data. +### Errors and Exit Status + +Every error is reported on standard error as a single line, and sqawk exits +with status 1: + +``` +$ sqawk -s "SELECT nmae FROM employees" employees.csv +sqawk: no such column: nmae +$ sqawk -s "SELECT 1" missing.csv +sqawk: cannot open 'missing.csv': No such file or directory +``` + +Command-line usage errors (an unknown option, a missing argument, an invalid +`-F` value) are reported by the argument parser with a usage hint and exit +status 2. A run that succeeds exits with status 0. In the REPL, an error is +printed in the same `sqawk: ` form and the session continues; see +[Interactive Mode](#interactive-mode--i) for the REPL's exit status. ### Help (--help) @@ -389,15 +560,15 @@ Sqawk supports various delimiter-separated file formats: File format detection follows these rules: 1. If a specific delimiter is provided with `-F`, it's used regardless of file extension -2. Files with a `.csv` extension use comma as the default delimiter -3. Files with **any other** extension -- `.tsv`, `.txt`, `.log` -- default to tab -4. Files with **no** extension default to comma, as does standard input +2. Files with a `.tsv` or `.tab` extension (any case) use tab +3. Every other file -- `.csv`, `.txt`, `.log`, no extension -- uses comma, as + does standard input #### Comment Support -Delimited files other than `.csv` may contain comment lines. Lines that begin -with `#` are ignored when such a file is read. In a `.csv` file, a `#` line is -an ordinary record: +By default a line starting with `#` is an ordinary record. With +`--comments`, lines that begin with `#` are skipped when files are read (any +file type), and `--write` puts them back where they were: ``` # This line is a comment and will be ignored @@ -418,10 +589,31 @@ Comment support is useful for: - A row with **too many** fields is an error naming the line, since there is no column to hold the extra values and dropping them would lose data +In `short.csv`, line 2 has two fields and loads as `1`, `2`, NULL: + +```csv +a,b,c +1,2 +4,5,6 +``` + +``` +$ sqawk -s "SELECT a, b, c, c IS NULL FROM short" short.csv +a,b,c,c IS NULL +1,2,,1 +4,5,6,0 +``` + +In `long.csv`, line 2 has four fields, and the file does not load: + ```csv a,b,c -1,2 # loads as 1, 2, NULL -1,2,3,4 # error: record has 4 fields, but the table has 3 columns +1,2,3,4 +``` + +``` +$ sqawk -s "SELECT * FROM long" long.csv +sqawk: long.csv: line 2: record has 4 fields, but the table has 3 columns ``` Padding is silent. If a query returns unexpected NULLs in the last columns, @@ -429,7 +621,11 @@ check the input for short rows. Fields are read per RFC 4180: a quoted field may contain the delimiter, line breaks, and doubled quotes (`"say ""hi"""`). Fields are never trimmed, and a -UTF-8 byte order mark at the start of the file is ignored. +UTF-8 byte order mark at the start of the file is not part of the first +column name (it is kept on `--write`). A quoted field whose closing quote is +missing is an error (*"FILE: line N: unterminated quoted field"*, N being the +line where the field starts), since +otherwise the rest of the file would load as one field. Completely empty input is a different matter. The first line is the header, so a file or pipe with no content has no column names and is rejected with *"has @@ -437,11 +633,13 @@ no header row"* rather than loading an unusable table. ### Defining Schemas with CREATE TABLE -While Sqawk automatically infers types from input files, you can explicitly define table schemas using the CREATE TABLE statement. This is useful for: +Besides the tables loaded from input files, you can create tables with the +CREATE TABLE statement. This is useful for: - Creating empty tables without loading from a file -- Ensuring specific column types for data validation -- Defining output file formats for new tables +- Declaring the column names and types of an existing file (see + [An Existing File](#an-existing-file)) +- Defining the file, and its delimiter, that a new table is written to #### Basic CREATE TABLE Syntax @@ -451,8 +649,8 @@ CREATE TABLE table_name ( column2 data_type, ... ) [LOCATION 'file_path'] - [STORED AS file_format] - [WITH (option_name='option_value', ...)] + [STORED AS TEXTFILE] + [WITH (DELIMITER='...', HEADER=true|false, COMMENTS=true|false)] ``` Example: @@ -471,12 +669,13 @@ CREATE TABLE employees ( #### Supported Data Types -Sqawk supports these data types in CREATE TABLE statements: - -- `INT` or `INTEGER`: For whole numbers -- `FLOAT` or `REAL`: For decimal numbers -- `TEXT` or `STRING`: For text values -- `BOOLEAN`: For true/false values +Any type name is accepted (`INT`, `BIGINT`, `VARCHAR(20)`, `DECIMAL(10,2)`, +`FLOAT`, `TEXT`, `BOOLEAN`, ...), and `.schema` shows each column's type as +declared. Every column needs a type: `CREATE TABLE t (a)` is a parse error. +Column types are not enforced: as in a loaded table, each value keeps its own +type, so `INSERT INTO t VALUES ('abc')` stores the text `abc` in an `INT` +column. See [Data Types](sql_reference.md#data-types) for how a type name +maps to an affinity. #### Setting File Location and Format @@ -490,7 +689,9 @@ CREATE TABLE logs ( ) LOCATION './logs/app.log' ``` -Currently, only TEXTFILE format is supported: +TEXTFILE is the only format: `STORED AS PARQUET` (or any other format) is an +error. Likewise `WITH` takes only `DELIMITER`, `HEADER` and `COMMENTS`; any +other option is an error: ```sql CREATE TABLE users ( @@ -502,7 +703,11 @@ CREATE TABLE users ( #### Specifying Custom Delimiters -For non-CSV formats, specify the delimiter with the WITH clause: +Without a WITH clause, the file gets the delimiter it will be loaded with: tab +for a `.tsv` or `.tab` path, comma for any other. So a table created with +`LOCATION './out.tsv'` is written tab-separated and reads back as the same +columns. For any other delimiter, give it with the WITH clause, which wins +over the extension: ```sql CREATE TABLE server_logs ( @@ -515,6 +720,72 @@ CREATE TABLE server_logs ( WITH (DELIMITER='\t') ``` +#### An Existing File + +When the `LOCATION` file already exists, the declared columns are its schema +(as for a Hive external table): the file is loaded under the declared names, +by the same loader as an input file, and its rows are the table's rows. + +- The file's first line is a header. It is skipped as data, it must have + exactly one field per declared column, and `--write` writes it back as it + was read: the declared names do not replace its text. +- `WITH (HEADER=false)` says the file has no header line: the first line is + data, and the file is written back without one. +- `WITH (COMMENTS=true)` skips `#` lines and keeps them in place, as + `--comments` does for input files. +- The delimiter follows the same rule as for a new file: `WITH + (DELIMITER=...)`, else tab for `.tsv` / `.tab`, else comma. +- Loading the file is not a change. With `--write`, the file is rewritten + only if a statement changed the table, and every row no statement changed + is written back byte for byte. + +```sh +$ cat people.csv +num,label +1,alice +2,bob +$ sqawk -s "CREATE TABLE p (id INT, name TEXT) LOCATION 'people.csv'; + INSERT INTO p VALUES (3, 'carol'); + SELECT * FROM p WHERE id > 1" seed.csv --write +id,name +2,bob +3,carol +$ cat people.csv +num,label +1,alice +2,bob +3,carol +``` + +A headerless file, here semicolon-separated: + +```sh +$ cat pairs.txt +1;alice +2;bob +$ sqawk -s "CREATE TABLE q (id INT, name TEXT) LOCATION 'pairs.txt' + WITH (DELIMITER=';', HEADER=false); + SELECT name FROM q WHERE id = 2" seed.csv +name +bob +``` + +A header whose field count differs from the declared columns is an error, +and so is a file that is already loaded, as an input file or by another +`CREATE TABLE ... LOCATION`: + +```sh +$ sqawk -s "CREATE TABLE p (id INT) LOCATION 'people.csv'" seed.csv +sqawk: people.csv: line 1: header has 2 fields, but the table declares 1 columns +$ sqawk -s "CREATE TABLE p (id INT, name TEXT) LOCATION 'people.csv'" people.csv +sqawk: file people.csv is already loaded as table people +``` + +A relative `LOCATION` is resolved against the current directory when the +statement runs. A `LOCATION` that does not exist yet is a new, empty table, +created by `--write` (with a header line of the declared names, or none +under `HEADER=false`). + #### Working with Created Tables After creating a table, you can insert data and query it: @@ -530,7 +801,9 @@ INSERT INTO products VALUES (1, 'Keyboard', 49.99), (2, 'Mouse', 29.99); SELECT * FROM products WHERE price < 40; ``` -Use the `--write` flag to save changes to the specified location: +A table created without `LOCATION` (like `products` above, or any `CREATE +TABLE ... AS SELECT`) is a scratch table: `--write` skips it. Use `LOCATION` +and the `--write` flag to save a new table to a file: ```sh sqawk -s "CREATE TABLE data (id INT, value FLOAT) LOCATION './output.csv'; @@ -587,6 +860,8 @@ Sqawk follows a safe-by-default approach to file modification: - Column and row order remain the same - NULL is written as an empty field - The file keeps its permissions; a symlink keeps pointing at the rewritten file + - Hard links to the file are broken: the file is replaced, not rewritten in place +- A read-only target file fails the whole run before any file is written Example of safe write behavior: @@ -597,9 +872,9 @@ sqawk -s "DELETE FROM data WHERE code NOT IN (SELECT code FROM lookup)" \ data.csv lookup.csv --write ``` -`UPDATE ... FROM other_table` is **not** supported, and is silently accepted -without doing anything -- no error, no rows changed. A cross-table update has -to go through a correlated subquery in the SET expression: +`UPDATE ... FROM other_table` is **not** supported (it is an error). A +cross-table update has to go through a correlated subquery in the SET +expression: ```sh sqawk -s "UPDATE data SET category = @@ -610,6 +885,26 @@ sqawk -s "UPDATE data SET category = A row with no matching lookup row gets NULL (written back as an empty field); add `WHERE code IN (SELECT code FROM lookup)` to leave such rows untouched. +### Transactions + +`BEGIN` opens a transaction, `COMMIT` (or `END`) keeps its changes and +`ROLLBACK` puts every table back as it was at `BEGIN`, as in SQLite. The +statements may come in one `-s`, several `-s` / `-f` arguments or separate +REPL lines: + +```sh +sqawk -s "BEGIN" -s "DELETE FROM data WHERE year < 2020" \ + -s "SELECT COUNT(*) FROM data" -s "ROLLBACK" data.csv --write +``` + +- `BEGIN DEFERRED`, `IMMEDIATE` and `EXCLUSIVE` all act as `BEGIN` +- `BEGIN` inside a transaction, and `COMMIT` / `ROLLBACK` outside one, are errors +- A transaction still open when the statements run out (or the REPL exits) + is rolled back, and with `--write` nothing it changed is written +- In the REPL with `.write on`, nothing is saved until `COMMIT`; `.save` and + `.load` are refused inside a transaction +- Savepoints (`SAVEPOINT`, `RELEASE`, `ROLLBACK TO`) are not supported + ## Examples ### Basic Queries @@ -652,7 +947,7 @@ sqawk -s "SELECT DISTINCT category FROM data" data.csv sqawk -s "SELECT COUNT(DISTINCT category) FROM data" data.csv # Unique combinations -sqawk -s "SELECT DISTINCT department, role FROM employees" employees.csv +sqawk -s "SELECT DISTINCT status, category FROM data" data.csv ``` ### Joins @@ -674,6 +969,76 @@ sqawk -s "SELECT u.name, p.name AS product, o.date users.csv orders.csv products.csv ``` +### Common Table Expressions (WITH) + +`WITH` names a query for the statement that follows, which reads it like a +table: + +```sh +sqawk -s "WITH big AS (SELECT * FROM employees WHERE salary > 70000) + SELECT department, COUNT(*) FROM big GROUP BY department" employees.csv +``` + +``` +department,COUNT(*) +Engineering,1 +Sales,2 +``` + +A CTE that names itself in a `UNION ALL` arm is recursive: the first arm +seeds the rows, and the second runs on each new row until it produces no +more. Given `staff.csv`: + +``` +id,name,manager +1,Ann, +2,Bob,1 +3,Cid,1 +4,Dee,2 +5,Eve,4 +``` + +```sh +# Count without a table +sqawk -s "WITH RECURSIVE n(i) AS (SELECT 1 UNION ALL SELECT i + 1 FROM n WHERE i < 5) + SELECT i, i * i AS square FROM n" staff.csv + +# Walk the reporting chain from the top, breadth-first +sqawk -s "WITH RECURSIVE chain(id, name, depth, path) AS ( + SELECT id, name, 0, name FROM staff WHERE manager IS NULL + UNION ALL + SELECT s.id, s.name, c.depth + 1, c.path || ' > ' || s.name + FROM staff s JOIN chain c ON s.manager = c.id) + SELECT name, depth, path FROM chain" staff.csv +``` + +``` +name,depth,path +Ann,0,Ann +Bob,1,Ann > Bob +Cid,1,Ann > Cid +Dee,2,Ann > Bob > Dee +Eve,3,Ann > Bob > Dee > Eve +``` + +Add `ORDER BY 3 DESC` inside the CTE (after its last arm) to walk depth-first +instead: Ann, Bob, Dee, Eve, Cid. WITH also drives modifications -- this +removes Bob and everyone under him: + +```sh +sqawk -s "WITH RECURSIVE under(id) AS ( + SELECT id FROM staff WHERE name = 'Bob' + UNION ALL + SELECT s.id FROM staff s JOIN under u ON s.manager = u.id) + DELETE FROM staff WHERE id IN (SELECT id FROM under)" staff.csv --write +``` + +A recursion that nothing stops runs forever; end it with a WHERE condition +in the recursive arm, or with `LIMIT` on the CTE body +(`... UNION ALL SELECT i + 1 FROM n LIMIT 10`). See the +[SQL reference](sql_reference.md#with-common-table-expressions) for the +full rules. + ### Data Modification ```sh @@ -683,10 +1048,13 @@ sqawk -s "UPDATE data SET status = 'archived' WHERE date < '2023-01-01'" data.cs # Delete rows sqawk -s "DELETE FROM data WHERE status = 'expired'" data.csv --write -# Insert new rows -sqawk -s "INSERT INTO data VALUES (100, 'New Item', 'active')" data.csv --write +# Insert a row; columns not listed are NULL (an empty field in the file) +sqawk -s "INSERT INTO data (id, name, status) VALUES (100, 'New Item', 'active')" data.csv --write ``` +`INSERT INTO data VALUES (...)` without a column list needs exactly one value +per column of `data`; any other count is an error. + ### String Functions ```sh @@ -712,8 +1080,9 @@ from SQLite" in the [SQL reference](sql_reference.md)): becomes a real, and division by zero is NULL. - `LIKE` is case-insensitive for ASCII letters; `ILIKE` folds all of Unicode. - String functions count characters, not bytes; `'a' || NULL` is NULL. -- Dates are UTC: `DATE('now')`, `DATE(d, '+1 month')`, `STRFTIME('%Y', d)`. -- Syntax that parses but is not implemented (`RETURNING`, `WITH`, +- Dates are UTC unless converted with `'localtime'`: `DATE('now')`, + `DATE(d, '+1 month')`, `STRFTIME('%Y', d)`. +- Syntax that parses but is not implemented (`RETURNING`, `UPDATE ... FROM`, ...) is an error, never silently ignored. ### System Files (--tabledef) @@ -724,10 +1093,8 @@ sqawk -F: --tabledef=passwd:user,pass,uid,gid,gecos,home,shell \ -s "SELECT user, home FROM passwd WHERE uid >= 1000" \ passwd=/etc/passwd -# Query /etc/hosts -sqawk --tabledef=hosts:ip,hostname \ - -s "SELECT * FROM hosts WHERE ip LIKE '192.168.%'" \ - hosts=/etc/hosts +# The same without names: columns a, b, c, ... +sqawk -F: --no-header -s "SELECT a, f FROM passwd WHERE c >= 1000" /etc/passwd ``` ### Output Redirection @@ -744,9 +1111,14 @@ sqawk -s "SELECT DISTINCT * FROM data" data.csv > deduped.csv sqawk -s "SELECT * FROM data" data.tsv > data.csv ``` -The output delimiter is whatever `-F` says, and comma when `-F` is absent. It -does not follow the input file's delimiter, which is what makes the conversion -above work. +The output delimiter is whatever `--output-delimiter` says, else `-F`, and +comma when neither is given. It does not follow the input file's delimiter, +which is what makes the conversion above work. `--no-output-header` drops +the header line, for appending to an existing file: + +```sh +sqawk --no-output-header -s "SELECT * FROM today" today.csv >> all.csv +``` Output is a well-formed delimited file: a field containing the delimiter, a quote or a line break, or with leading or trailing spaces, is quoted @@ -798,59 +1170,70 @@ sqawk> .tables Tables: customers sales - sqawk> .schema sales CREATE TABLE sales ( id TEXT, customer_id TEXT, amount TEXT ); - sqawk> SELECT COUNT(*) FROM sales; -COUNT +COUNT(*) 1250 - sqawk> SELECT c.name, SUM(s.amount) AS total FROM customers c JOIN sales s ON c.id = s.customer_id GROUP BY c.name ORDER BY total DESC LIMIT 3; name,total Enterprise Corp,58750.25 Acme Inc,45620.75 -Globex,31004.10 - +Globex,31004.1 sqawk> .exit ``` -An unaliased aggregate takes the function's name as its column header -- -`COUNT`, `SUM`, `AVG` -- so use `AS` when the header matters downstream. +An unaliased expression takes its SQL text as its column header -- +`COUNT(*)`, `SUM(s.amount)` -- and a column reference its bare column name +(`c.name` is headed `name`), as in SQLite; use `AS` when the header matters +downstream. ## Working with Large Files -Sqawk holds a whole table in memory at once, so the dataset must fit, which is -worth planning for with large files: +Sqawk reads every input file whole and holds all of its tables in memory for +the whole run, so the data must fit in RAM. + +**How much memory.** A loaded table takes roughly six to seven times its +file's size: the parsed values, plus the file's own bytes, which are kept so +that `--write` can reproduce untouched rows exactly. Each statement's result +set is also built in memory before it is printed, so a query that returns +most of a table needs about as much again. Measured on an 83 MB CSV of one +million 10-column rows: + +| Query | Peak memory | +|-------|-------------| +| `SELECT COUNT(*) FROM big` | 530 MB | +| `SELECT * FROM big WHERE score > 95` (5% of rows) | 580 MB | +| `SELECT city, COUNT(*) FROM big GROUP BY city` | 660 MB | +| `SELECT * FROM big` | 1.4 GB | +| `SELECT * FROM big ORDER BY amount` | 1.9 GB | **Tips for handling large files:** -1. **Filter early**: When possible, use WHERE clauses to reduce the working dataset +1. **Return less**: the table is loaded in full whatever the query, but a + WHERE clause, an aggregate or a LIMIT keeps the result set small ```sh sqawk -s "SELECT * FROM large_data WHERE date > '2023-01-01'" large_data.csv ``` -2. **Select only needed columns**: Minimize memory usage by selecting only required columns +2. **Process in batches**: split a file that does not fit and process the + pieces (each piece needs the header line) ```sh - sqawk -s "SELECT id, name FROM large_data" large_data.csv - ``` - -3. **Process in batches**: Split large files and process them in segments - ```sh - # Process first using head command (Unix/Linux) - head -n 1000000 large_data.csv > batch1.csv + # First million data rows (Unix/Linux) + head -n 1000001 large_data.csv > batch1.csv sqawk -s "SELECT * FROM batch1 WHERE value > 100" batch1.csv ``` -4. **Monitor memory usage**: Particularly when joining large tables, be aware of memory constraints +3. **Joins**: an equality join builds a hash index on its inner table, so it + needs memory for that index on top of both tables, and its result set grows + with the number of matches ```sh - # Using a more targeted join condition reduces memory requirements sqawk -s "SELECT a.id, b.name FROM large_a a INNER JOIN large_b b ON a.id = b.id WHERE a.region = 'West'" large_a.csv large_b.csv ``` @@ -858,7 +1241,7 @@ worth planning for with large files: **Common Issues and Solutions:** -1. **"Table not found" error**: +1. **"no such table" error**: - Check that the filename matches the table name in your SQL - If using custom table names, verify the syntax: `tablename=filename.csv` - Ensure file paths are correct and files are accessible @@ -884,23 +1267,28 @@ worth planning for with large files: - `COUNT(*)` counts rows while `COUNT(col)` counts non-NULL values 5. **CSV parsing errors with malformed rows**: - - Error messages about "field count mismatch" indicate rows with inconsistent numbers of fields - - Error messages include line numbers to help locate problematic rows - - Common causes include: - - Missing fields or extra delimiters - - Improperly escaped quotes inside fields - - Newlines within quoted fields - - Use the error recovery options described in the File Format Support section to handle malformed rows + - Load errors have the form `FILE: line N: MESSAGE` + - *"record has N fields, but the table has M columns"* means a row has + more fields than the header + - *"unterminated quoted field"* means a quote opened at the reported line + is never closed + - Common causes include extra delimiters, improperly escaped quotes inside + fields, and the wrong delimiter (`.tsv`/`.tab` are tab-separated, every + other file comma-separated, unless `-F` is given) + - sqawk does not skip malformed rows: fix the reported line 6. **Issues with comment lines**: - - Comments must start at the beginning of a line with the comment character - - Comment characters appearing within data (not at the start of a line) are treated as regular data - - If you're seeing unexpected parsing errors, check if comment lines are properly formatted + - Lines starting with `#` are data unless `--comments` is given; a `#` + header line or `#` rows in a file meant to have comments usually means + the flag is missing + - Comments must start at the beginning of a line; a `#` anywhere else is + regular data 7. **Memory limitations**: - - If processing very large files, filter data early in your queries - - Consider processing in batches or using more targeted queries - - Select only the columns you need rather than using SELECT * + - Every input table is loaded whole: expect six to seven times the file + size, plus the result set (see [Working with Large Files](#working-with-large-files)) + - Keep result sets small with WHERE, aggregates or LIMIT + - Split a file that does not fit and process the pieces 8. **Changes not saved**: - Remember to use the `--write` flag to save changes @@ -909,7 +1297,8 @@ worth planning for with large files: 9. **SQL syntax errors**: - Try running your query in interactive mode to get immediate feedback - - Use the `-v` verbose flag to see the exact SQL being executed + - The error message names the token where parsing failed, with its line + and column - Verify SQL statement syntax, particularly quotes, parentheses, and required clauses 10. **Special characters in files**: @@ -948,8 +1337,3 @@ For more help, use the verbose mode (`-v`) to see detailed information about pro - [SQL Language Reference](sql_reference.md) - Complete guide to Sqawk's SQL dialect - [GitHub Repository](https://github.com/jgarzik/sqawk) - Source code and issue tracking -- [Release Notes](https://github.com/jgarzik/sqawk/releases) - Latest features and bug fixes - ---- - -*This user guide describes Sqawk as of its current version. Features and behavior may change in future releases.* \ No newline at end of file diff --git a/src/capacity.rs b/src/capacity.rs index 0389876..211babb 100644 --- a/src/capacity.rs +++ b/src/capacity.rs @@ -60,29 +60,10 @@ pub const DEFAULT_TABLE_CAPACITY: usize = 8; /// Grows dynamically, but start with reasonable size. pub const DEFAULT_SORTER_ROWS_CAPACITY: usize = 256; -/// Default capacity for sort key vectors. -pub const DEFAULT_SORT_KEYS_CAPACITY: usize = 4; - // ============================================================================= -// Headerless File Detection +// Headerless files // ============================================================================= -/// Check if a field looks like data rather than a header name. -/// Used to auto-detect headerless files like /etc/passwd. -#[inline] -pub fn is_data_field(field: &str) -> bool { - field.starts_with('/') || // Path - field == "*" || // Password placeholder - field == "root" || // Common username - field == "nobody" || // Common username - field.parse::().is_ok() // Numeric ID -} - -/// Check if a row of fields looks like data rather than headers. -pub fn is_likely_data_row>(fields: &[S]) -> bool { - fields.iter().any(|f| is_data_field(f.as_ref())) -} - /// Generate alphabetical column names (a, b, c, ..., z, aa, ab, ...). pub fn generate_alpha_columns(count: usize) -> Vec { (0..count) diff --git a/src/cli.rs b/src/cli.rs index cf345a2..ec66ac8 100644 --- a/src/cli.rs +++ b/src/cli.rs @@ -5,13 +5,13 @@ //! enabling users to specify SQL statements, input files, and processing options. //! //! Key features of the CLI: -//! - Support for multiple SQL statements in a single invocation +//! - SQL given inline (`-s`) or read from files (`-f`), run in command-line order //! - Flexible file specification with optional table name overrides //! - Opt-in file modification with the --write flag -//! - Diagnostic output control via the --verbose flag +//! - Output delimiter and header control +//! - Debugging output via the --verbose flag -use anyhow::Result; -use clap::Parser; +use clap::{CommandFactory, FromArgMatches, Parser}; /// Command-line arguments for sqawk /// @@ -34,7 +34,12 @@ use clap::Parser; or after the name given as [table_name=]file_path. Statements run in \ the order given, and each one sees the effects of those before it.\n\n\ Input files are never modified unless --write is given, and then only \ - the tables an INSERT, UPDATE or DELETE actually changed." + the tables that INSERT, UPDATE, DELETE, ALTER TABLE or TRUNCATE changed \ + (and tables created with CREATE TABLE ... LOCATION). BEGIN / COMMIT / \ + ROLLBACK group statements; a transaction still open at the end is \ + rolled back before anything is written.\n\n\ + Errors are reported on standard error as one line, `sqawk: `, \ + with exit status 1; command-line usage errors exit with status 2." )] pub struct SqawkArgs { /// SQL statements to execute @@ -45,17 +50,36 @@ pub struct SqawkArgs { #[clap( short, long, - required_unless_present = "interactive", - help = "SQL statement to execute" + value_name = "SQL", + required_unless_present_any = ["interactive", "file"], + help = "SQL statement(s) to execute (repeatable)" )] pub sql: Vec, + /// Files of SQL statements to execute + /// + /// Each file may hold several `;`-separated statements. `-f -` reads the + /// statements from standard input. `-s` and `-f` may be mixed; they run + /// in the order given on the command line. + #[clap( + short = 'f', + long = "file", + value_name = "FILE", + help = "Read SQL statements from FILE ('-' for stdin; repeatable, runs in order with -s)" + )] + pub file: Vec, + /// Start in interactive mode (REPL) /// /// When enabled, launches an interactive shell for entering SQL commands. /// This mode allows exploration of data with immediate feedback. /// Type .help in the interactive shell to see available commands. - #[clap(short, long, help = "Start in interactive mode")] + #[clap( + short, + long, + conflicts_with_all = ["sql", "file"], + help = "Start in interactive mode (cannot be combined with -s or -f)" + )] pub interactive: bool, /// Input files to process - format: [table_name=]file_path @@ -88,12 +112,40 @@ pub struct SqawkArgs { /// Specify field separator character /// - /// Similar to awk's -F option, this sets the field separator for all input files. - /// Default behavior is to use commas for .csv files and tabs for other file types. - /// Examples: -F: for colon-separated files, -F\\t for tab-separated files. - #[clap(short = 'F', help = "Field separator character")] + /// Similar to awk's -F option, this sets the field separator for all input files, + /// and the output delimiter unless --output-delimiter is given. + /// Without it, `.tsv` and `.tab` files are tab-separated and every other file + /// (and standard input) is comma-separated. + /// Examples: -F: for colon-separated files, -F'\t' for tab-separated files. + #[clap( + short = 'F', + long = "field-separator", + value_name = "SEP", + value_parser = parse_separator, + help = "Input field separator, one character or \\t (also the output delimiter by default)" + )] pub field_separator: Option, + /// Delimiter between fields of query results + /// + /// Defaults to the -F separator when one is given, else a comma. Same + /// rules as -F: one ASCII character other than a quote or line break, or + /// `\t` for tab. + #[clap( + long = "output-delimiter", + value_name = "SEP", + value_parser = parse_separator, + help = "Delimiter for query results (default: the -F separator, else comma)" + )] + pub output_delimiter: Option, + + /// Omit the header line of every result set + #[clap( + long = "no-output-header", + help = "Omit the header line of query results" + )] + pub no_output_header: bool, + /// Input files have no header row /// /// The first line of every input file is data. Columns are named a, b, @@ -101,11 +153,23 @@ pub struct SqawkArgs { #[clap(long = "no-header", help = "Treat the first line of each file as data")] pub no_header: bool, - /// Enable verbose diagnostic output + /// Lines starting with `#` are comments + /// + /// Without this flag a line starting with `#` is an ordinary record. With + /// it, such lines are skipped when files are read, and kept in place when + /// a table is written back with --write. + #[clap(long, help = "Skip lines starting with '#' (kept on --write)")] + pub comments: bool, + + /// Print debugging information on standard error /// - /// When enabled, shows detailed information about SQL execution, - /// table loading, and modifications made to the data. - #[clap(short, long, help = "Enable verbose output")] + /// Shows table loading, the compiled bytecode and each executed + /// instruction. The output is for debugging; its format is not stable. + #[clap( + short, + long, + help = "Print debugging output on stderr (format is not stable)" + )] pub verbose: bool, /// Write changes back to input files (default is to not modify files) @@ -116,36 +180,86 @@ pub struct SqawkArgs { /// and their original file format and delimiters will be preserved. #[clap(short = 'w', long, help = "Write changes back to input files")] pub write: bool, + + /// Every `-s` and `-f`, in command-line order (filled in by [`parse_args`]). + #[clap(skip)] + pub script: Vec, +} + +/// One `-s` or `-f` argument. +#[derive(Debug, Clone, PartialEq)] +pub enum SqlSource { + /// `-s SQL`: the statement text itself + Text(String), + /// `-f FILE`: a file of statements (`-` is standard input) + File(String), } /// Parse command-line arguments into the SqawkArgs structure /// -/// This function uses clap to handle argument parsing, validation, and help text generation. -/// It automatically generates usage information, handles errors for missing required arguments, -/// and properly validates supported options. -/// -/// # Returns -/// * `Ok(SqawkArgs)` - Command-line arguments successfully parsed -/// * `Err` - Error during argument parsing (handled by clap, usually results in help text display) -pub fn parse_args() -> Result { - Ok(SqawkArgs::parse()) +/// clap handles usage errors itself (message on stderr, exit status 2). +/// The `-s` and `-f` values are merged into [`SqawkArgs::script`] by their +/// position on the command line. +pub fn parse_args() -> SqawkArgs { + let matches = SqawkArgs::command().get_matches(); + let mut args = match SqawkArgs::from_arg_matches(&matches) { + Ok(args) => args, + Err(e) => e.exit(), + }; + let mut script: Vec<(usize, SqlSource)> = Vec::new(); + if let Some(idx) = matches.indices_of("sql") { + script.extend(idx.zip(args.sql.iter().cloned().map(SqlSource::Text))); + } + if let Some(idx) = matches.indices_of("file") { + script.extend(idx.zip(args.file.iter().cloned().map(SqlSource::File))); + } + script.sort_by_key(|(i, _)| *i); + args.script = script.into_iter().map(|(_, s)| s).collect(); + args +} + +/// clap value parser for `-F` / `--output-delimiter`. +fn parse_separator(sep: &str) -> Result { + validate_separator(sep) } -/// Validate a `-F` field separator and return it in canonical form. +/// Validate a field separator and return it in canonical form. /// /// The separator must be exactly one ASCII byte, other than a quote or a /// line break; the two-character escape `\t` is accepted as a tab. -pub fn validate_separator(sep: &str) -> Result { +pub fn validate_separator(sep: &str) -> Result { let sep = if sep == "\\t" { "\t" } else { sep }; match sep.as_bytes() { - [] => anyhow::bail!("field separator (-F) must not be empty"), - [b'"' | b'\n' | b'\r'] => { - anyhow::bail!("field separator (-F) cannot be a quote or a line break") - } + [] => Err("separator must not be empty".to_string()), + [b'"' | b'\n' | b'\r'] => Err("separator cannot be a quote or a line break".to_string()), [b] if b.is_ascii() => Ok(sep.to_string()), - _ => anyhow::bail!( - "field separator (-F) must be a single ASCII character (or \\t for tab), got {:?}", + _ => Err(format!( + "separator must be a single ASCII character (or \\t for tab), got {:?}", sep - ), + )), + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn separators() { + assert_eq!(validate_separator("\\t").unwrap(), "\t"); + assert_eq!(validate_separator(":").unwrap(), ":"); + assert!(validate_separator("").is_err()); + assert!(validate_separator("\"").is_err()); + assert!(validate_separator("ab").is_err()); + } + + #[test] + fn sql_and_files_keep_command_line_order() { + let m = SqawkArgs::command() + .try_get_matches_from(["sqawk", "-s", "A", "-f", "x.sql", "-s", "B", "t.csv"]) + .unwrap(); + let idx: Vec = m.indices_of("sql").unwrap().collect(); + let fidx: Vec = m.indices_of("file").unwrap().collect(); + assert!(idx[0] < fidx[0] && fidx[0] < idx[1]); } } diff --git a/src/config.rs b/src/config.rs index 1a342fa..bcd870d 100644 --- a/src/config.rs +++ b/src/config.rs @@ -26,6 +26,15 @@ pub struct AppConfig { /// Treat the first line of every input file as data (`--no-header`) no_header: bool, + + /// Skip lines starting with `#` in input files (`--comments`) + comments: bool, + + /// Delimiter between fields of query results (`--output-delimiter`) + output_delimiter: Option, + + /// Omit the header line of every result set (`--no-output-header`) + no_output_header: bool, } impl AppConfig { @@ -48,9 +57,38 @@ impl AppConfig { table_definitions, write_changes, no_header: false, + comments: false, + output_delimiter: None, + no_output_header: false, } } + /// Set the delimiter for query results (`None`: follow -F, else comma) + pub fn with_output_delimiter(mut self, delimiter: Option) -> Self { + self.output_delimiter = delimiter; + self + } + + /// The delimiter for query results: `--output-delimiter`, else the -F + /// separator, else a comma. + pub fn output_delimiter(&self) -> String { + self.output_delimiter + .clone() + .or_else(|| self.field_separator.clone()) + .unwrap_or_else(|| ",".to_string()) + } + + /// Set whether result sets are printed without their header line + pub fn with_no_output_header(mut self, no_output_header: bool) -> Self { + self.no_output_header = no_output_header; + self + } + + /// Whether result sets are printed without their header line + pub fn no_output_header(&self) -> bool { + self.no_output_header + } + /// Set whether input files have no header row pub fn with_no_header(mut self, no_header: bool) -> Self { self.no_header = no_header; @@ -62,6 +100,17 @@ impl AppConfig { self.no_header } + /// Set whether lines starting with `#` are comments + pub fn with_comments(mut self, comments: bool) -> Self { + self.comments = comments; + self + } + + /// Whether lines starting with `#` are comments + pub fn comments(&self) -> bool { + self.comments + } + /// Get the verbose flag pub fn verbose(&self) -> bool { self.verbose diff --git a/src/csv_handler.rs b/src/csv_handler.rs index 2405e76..057af27 100644 --- a/src/csv_handler.rs +++ b/src/csv_handler.rs @@ -1,52 +1,42 @@ //! Delimited file reading and writing for sqawk //! -//! Every input -- `.csv`, TSV, `-F`-delimited files and standard input -- is -//! parsed here by the `csv` crate with one configuration: RFC 4180 quoting +//! Every input -- `.csv`, `.tsv`, `-F`-delimited files and standard input -- +//! is parsed here by the `csv` crate with one configuration: RFC 4180 quoting //! (`""` inside a quoted field is a literal quote), quoted fields may span -//! lines, fields are never trimmed, a UTF-8 byte order mark is stripped, and -//! rows may be ragged (short rows are padded with NULL). +//! lines, fields are never trimmed, and rows may be ragged (short rows are +//! padded with NULL). A quote that is never closed is an error. //! -//! Writeback goes through the same module, so that a file read and written -//! back without changes is byte-identical: cells are inferred with a rule -//! that round-trips (see `Value::from(&str)`), NULL is the empty field, and -//! the header/line-ending/comment layout of the source is recorded on the -//! table and reproduced, including the `#` comment lines themselves (see -//! `CommentLines`). +//! Writeback goes through the same module. Each table keeps the bytes it was +//! read from, so every row no statement changed is written back exactly as +//! it was read (quoting, padding, line ending, invalid UTF-8 and all), as are +//! the byte order mark, the header, blank lines and `#` comment lines (see +//! `RawRows` and `CommentLines`). Rows that were inserted or changed are +//! written with RFC 4180 quoting where a field needs it. use std::io::{BufWriter, Write}; use std::path::{Path, PathBuf}; use tempfile::NamedTempFile; -use crate::capacity::{generate_alpha_columns, is_likely_data_row}; +use crate::capacity::generate_alpha_columns; use crate::error::{SqawkError, SqawkResult}; -use crate::table::{CommentLines, FileLayout, Table, Value}; - -/// How the first record of a file is treated. -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub enum HeaderMode { - /// The first record is the header (the rule for `.csv` and stdin). - Present, - /// Every record is data; columns are named a, b, c, ... unless given. - Absent, - /// Guess from the first record (`is_likely_data_row`); used for - /// non-.csv delimited files such as /etc/passwd. - Guess, -} +use crate::table::{CommentLines, FileLayout, RawRows, RawSpan, Table, Value}; /// Options for reading one delimited file. #[derive(Debug, Clone)] pub struct ReadOptions { /// Field separator byte pub delimiter: u8, - /// Header policy - pub header: HeaderMode, - /// Skip lines that start with `#` + /// The first record is the header. When false, every record is data and + /// the columns are named a, b, c, ... unless `columns` names them. + /// When true and `columns` is given, the header is skipped as data (and + /// kept for writeback) but `columns` names the columns; the header must + /// have exactly one field per column. + pub header: bool, + /// Skip lines that start with `#` (`--comments`) pub comments: bool, - /// Column names to use; implies the file has no header row + /// Column names to use instead of the header's (see `header`) pub columns: Option>, - /// Skip or repair malformed records instead of failing - pub recover_errors: bool, } impl Default for ReadOptions { @@ -54,10 +44,9 @@ impl Default for ReadOptions { fn default() -> Self { ReadOptions { delimiter: b',', - header: HeaderMode::Present, + header: true, comments: false, columns: None, - recover_errors: false, } } } @@ -65,13 +54,18 @@ impl Default for ReadOptions { /// UTF-8 byte order mark const BOM: &[u8] = b"\xef\xbb\xbf"; -/// Field bytes to text. Invalid UTF-8 is replaced rather than rejected. +/// Field bytes to text. Invalid UTF-8 is replaced rather than rejected (the +/// row's source bytes are kept, so an untouched row is written back as read). fn field_text(bytes: &[u8]) -> std::borrow::Cow<'_, str> { String::from_utf8_lossy(bytes) } /// Parse delimited `data` into a table. /// +/// The table keeps `data` itself: every record's source bytes (see +/// `RawRows`), so writeback reproduces the rows no statement changed +/// byte-for-byte. +/// /// # Arguments /// * `table_name` - name of the resulting table /// * `data` - the whole file contents @@ -81,234 +75,414 @@ fn field_text(bytes: &[u8]) -> std::borrow::Cow<'_, str> { /// * `source` - file name used in error messages pub fn read_table( table_name: &str, - data: &[u8], + data: Vec, file_path: Option, delimiter: String, opts: &ReadOptions, source: &str, ) -> SqawkResult { - let data = data.strip_prefix(BOM).unwrap_or(data); + let bom = data.starts_with(BOM); + let base = if bom { BOM.len() } else { 0 }; + let body = &data[base..]; - // Line ending of the first record decides the writeback terminator. - let crlf = data + // Line ending of the first record decides the terminator of the records + // writeback generates. + let crlf = body .iter() .position(|&b| b == b'\n') - .is_some_and(|i| i > 0 && data[i - 1] == b'\r'); + .is_some_and(|i| i > 0 && body[i - 1] == b'\r'); let mut reader = csv::ReaderBuilder::new() .has_headers(false) .flexible(true) .delimiter(opts.delimiter) .comment(if opts.comments { Some(b'#') } else { None }) - .from_reader(data); + .from_reader(body); + let header = opts.header; let mut record = csv::ByteRecord::new(); - let mut line = 0usize; - let mut comment_lines = CommentLines::default(); - - // First record: the header, or the first data row. - let mut pending_first: Option> = None; - let first = read_record(&mut reader, &mut record, source, &mut line); - let mut has_first = first?; - if opts.comments { - let (lines, to_eof) = skipped_comments(data, &record); - comment_lines.head = lines; - has_first &= !(to_eof && is_phantom(&record)); - } - let header_mode = if opts.columns.is_some() { - HeaderMode::Absent - } else { - opts.header - }; - let (columns, has_header) = if !has_first { - (opts.columns.clone().unwrap_or_default(), false) - } else { - let fields: Vec = record.iter().map(|f| field_text(f).into_owned()).collect(); - let is_header = match header_mode { - HeaderMode::Present => true, - HeaderMode::Absent => false, - HeaderMode::Guess => !is_likely_data_row(&fields), + let mut lines = LineCounter::new(body); + // The bytes between records: blank lines, and `#` lines under + // --comments. Kept verbatim, anchored to the record that follows. + let mut gaps = CommentLines::default(); + let mut header_span = RawSpan::NONE; + let mut spans: Vec = Vec::new(); + let mut rows: Vec> = Vec::new(); + let mut columns: Option> = None; + let mut width = 0; + // Where the previous record's bytes end. + let mut prev_end = 0usize; + // The last record's span within `body`. + let mut last: Option<(usize, usize)> = None; + + loop { + let more = read_record(&mut reader, &mut record, source, &mut lines)?; + let from = record + .position() + .map_or(prev_end, |p| p.byte() as usize) + .clamp(prev_end, body.len()); + let start = record_start(body, from, opts.comments); + // A comment line that ends the input without a line ending comes + // back from the csv crate as an empty record; it is not a row. + if !more || start >= body.len() { + gaps.tail = body[prev_end..].to_vec(); + break; + } + let mut end = (reader.position().byte() as usize).clamp(start, body.len()); + // csv-core ends a CRLF record at the CR; the LF belongs to it too. + if end > start && end < body.len() && body[end - 1] == b'\r' && body[end] == b'\n' { + end += 1; + } + let line = lines.line_at(start); + let gap = &body[prev_end..start]; + prev_end = end; + last = Some((start, end)); + let span = RawSpan { + start: base + start, + end: base + end, }; - if is_header { - (fields, true) - } else { - let cols = opts - .columns - .clone() - .unwrap_or_else(|| generate_alpha_columns(fields.len())); - pending_first = Some(fields.into_iter().map(Value::from).collect()); - (cols, false) + + let Some(cols) = &columns else { + // The first record: the header, or the first row. + gaps.head = gap.to_vec(); + let cols = if header { + header_span = span; + match &opts.columns { + None => record.iter().map(|f| field_text(f).into_owned()).collect(), + Some(cols) if cols.len() == record.len() => cols.clone(), + Some(cols) => { + return Err(SqawkError::CsvParseError { + file: source.to_string(), + line, + error: format!( + "header has {} fields, but the table declares {} columns", + record.len(), + cols.len() + ), + }) + } + } + } else { + opts.columns + .clone() + .unwrap_or_else(|| generate_alpha_columns(record.len())) + }; + check_column_names(&cols, source, line)?; + width = cols.len(); + columns = Some(cols); + if header { + continue; + } + push_row(&mut rows, &record, width, source, line)?; + spans.push(span); + continue; + }; + debug_assert_eq!(cols.len(), width); + if !gap.is_empty() { + gaps.before + .entry(rows.len()) + .or_default() + .extend_from_slice(gap); + } + push_row(&mut rows, &record, width, source, line)?; + spans.push(span); + } + + // csv-core takes a quote that is never closed as running to the end of + // the input, so the rest of the file would load as one field. + if let Some((start, end)) = last { + if let Some(quote) = unclosed_quote(&body[start..end], opts.delimiter) { + return Err(SqawkError::CsvParseError { + file: source.to_string(), + line: lines.line_at(start + quote), + error: "unterminated quoted field".to_string(), + }); + } + } + + let columns = match columns { + Some(cols) => cols, + // No records at all (empty input, or only blank/comment lines). + None => { + let cols = opts.columns.clone().unwrap_or_default(); + check_column_names(&cols, source, 1)?; + gaps.head = std::mem::take(&mut gaps.tail); + cols } }; + drop(reader); let mut table = Table::new_with_delimiter(table_name, columns, file_path, delimiter); table.set_file_layout(FileLayout { - has_header, + // A file with no header line yet gets one of the given column names. + has_header: header && (header_span != RawSpan::NONE || opts.columns.is_some()), + bom, crlf, comments: opts.comments, }); - let width = table.column_count(); - let mut sink = RowSink { - rows: Vec::new(), - width, - skipped: 0, - recover: opts.recover_errors, - source, + // Loading is not a modification. + table.replace_rows(rows); + table.set_raw(RawRows { + buf: data, + header: header_span, + rows: spans, + }); + table.set_comment_lines(gaps); + table.mark_unmodified(); + Ok(table) +} + +/// Reject a header (or column list) with an empty or repeated name. Names +/// are case-insensitive, so `a` and `A` are the same column. +fn check_column_names(cols: &[String], source: &str, line: usize) -> SqawkResult<()> { + let err = |error: String| SqawkError::CsvParseError { + file: source.to_string(), + line, + error, }; - if let Some(first) = pending_first { - sink.push(first, line)?; + if let Some(i) = cols.iter().position(String::is_empty) { + return Err(err(format!("column {} has an empty name", i + 1))); } - - loop { - let mut next = read_record(&mut reader, &mut record, source, &mut line); - if opts.comments { - // The comment lines skipped before this record, or before the - // end of the file, go in front of the next row to be pushed (or - // into the tail, past the last row). - let (lines, to_eof) = skipped_comments(data, &record); - if !lines.is_empty() { - comment_lines - .before - .entry(sink.rows.len()) - .or_default() - .extend_from_slice(&lines); - } - if to_eof && matches!(next, Ok(true)) && is_phantom(&record) { - next = Ok(false); - } - } - match next { - Ok(true) => { - let row: Vec = record - .iter() - .map(|f| match std::str::from_utf8(f) { - Ok(s) => Value::from(s), - Err(_) => Value::from(field_text(f).into_owned()), - }) - .collect(); - sink.push(row, line)?; - } - Ok(false) => break, - Err(e) if opts.recover_errors => { - sink.skipped += 1; - eprintln!("Warning: Skipping malformed row: {}", e); + let mut seen = std::collections::HashSet::with_capacity(cols.len()); + let mut reported = std::collections::HashSet::new(); + let mut dups: Vec<&str> = Vec::new(); + for name in cols { + let key = name.to_ascii_lowercase(); + if seen.contains(&key) { + if reported.insert(key) { + dups.push(name); } - Err(e) => return Err(e), + } else { + seen.insert(key); } } - - if sink.skipped > 0 { - eprintln!( - "Note: Skipped {} malformed rows while loading {}", - sink.skipped, source - ); + if !dups.is_empty() { + return Err(err(format!("duplicate column name: {}", dups.join(", ")))); } - - // Loading is not a modification. - table.replace_rows(sink.rows); - table.set_comment_lines(comment_lines); - table.mark_unmodified(); - Ok(table) + Ok(()) } -/// Collects loaded rows, enforcing the table width. -struct RowSink<'a> { - rows: Vec>, +/// Append `record` as a row of `width` columns. A short record is padded +/// with NULL; a long one is an error. +fn push_row( + rows: &mut Vec>, + record: &csv::ByteRecord, width: usize, - skipped: usize, - recover: bool, - source: &'a str, + source: &str, + line: usize, +) -> SqawkResult<()> { + if record.len() > width { + return Err(SqawkError::CsvParseError { + file: source.to_string(), + line, + error: format!( + "record has {} fields, but the table has {} columns", + record.len(), + width + ), + }); + } + let mut row: Vec = Vec::with_capacity(width); + row.extend(record.iter().map(|f| match std::str::from_utf8(f) { + Ok(s) => Value::from(s), + Err(_) => Value::from(field_text(f).into_owned()), + })); + // A short record is padded: the missing trailing fields are NULL. + row.resize(width, Value::Null); + rows.push(row); + Ok(()) } -impl RowSink<'_> { - fn push(&mut self, mut row: Vec, line: usize) -> SqawkResult<()> { - if row.len() > self.width { - if self.recover { - self.skipped += 1; - eprintln!( - "Warning: Skipping row at line {} of {}: {} fields, expected {}", - line, - self.source, - row.len(), - self.width - ); - return Ok(()); +/// Where the record the csv crate reads from `from` begins: past the blank +/// lines (CR and LF bytes) and, with `comments`, the `#` lines (each running +/// to its LF) that csv-core skips first. +fn record_start(data: &[u8], from: usize, comments: bool) -> usize { + let mut i = from; + while i < data.len() { + match data[i] { + b'\r' | b'\n' => i += 1, + b'#' if comments => { + i = data[i..] + .iter() + .position(|&b| b == b'\n') + .map_or(data.len(), |n| i + n + 1); } - return Err(SqawkError::CsvParseError { - file: self.source.to_string(), - line, - error: format!( - "record has {} fields, but the table has {} columns", - row.len(), - self.width - ), - }); + _ => break, } - // A short record is padded: the missing trailing fields are NULL. - row.resize(self.width, Value::Null); - self.rows.push(row); - Ok(()) } + i } -/// The `#` comment lines the reader skipped before `record` (which may be the -/// empty record of end of input), verbatim with their line endings. -/// -/// A record's position is where the reader stood when it started on it: -/// just past the previous record. What lies between there and the record -/// itself is blank lines (or the rest of the previous record's line ending) -/// and comment lines; the scan stops at the first line that is neither. +/// If `rec` (one record's bytes) ends inside a quoted field, the offset of +/// that field's opening quote. Mirrors csv-core's quoting: a quote opens a +/// quoted field only at the start of a field, `""` inside one is a literal +/// quote, and after the closing quote the field continues unquoted up to the +/// delimiter. +fn unclosed_quote(rec: &[u8], delimiter: u8) -> Option { + #[derive(PartialEq)] + enum S { + FieldStart, + Unquoted, + Quoted, + QuoteInQuoted, + } + let mut state = S::FieldStart; + let mut opened = 0; + for (i, &b) in rec.iter().enumerate() { + state = match state { + S::FieldStart if b == b'"' => { + opened = i; + S::Quoted + } + S::FieldStart | S::Unquoted | S::QuoteInQuoted if b == delimiter => S::FieldStart, + S::FieldStart | S::Unquoted | S::QuoteInQuoted if b == b'\r' || b == b'\n' => { + S::FieldStart + } + S::FieldStart | S::Unquoted => S::Unquoted, + S::Quoted if b == b'"' => S::QuoteInQuoted, + S::Quoted => S::Quoted, + S::QuoteInQuoted if b == b'"' => S::Quoted, + S::QuoteInQuoted => S::Unquoted, + }; + } + (state == S::Quoted).then_some(opened) +} + +/// 1-based line numbers of byte offsets in a file body, for error messages. /// -/// Also returns whether the comment lines run to the end of the input. -fn skipped_comments(data: &[u8], record: &csv::ByteRecord) -> (Vec, bool) { - let Some(pos) = record.position() else { - return (Vec::new(), false); - }; - let mut i = (pos.byte() as usize).min(data.len()); - let mut out = Vec::new(); - while i < data.len() { - let end = data[i..] - .iter() - .position(|&b| b == b'\n') - .map_or(data.len(), |n| i + n + 1); - let line = &data[i..end]; - if line[0] == b'#' { - out.extend_from_slice(line); - } else if !line.iter().all(|&b| b == b'\r' || b == b'\n') { - break; +/// The line is counted from the bytes themselves, not taken from csv-core's +/// record positions: csv-core ends a CRLF record at the CR and counts its LF +/// only when the next record is read, so its line numbers lag by one in CRLF +/// files. A line ends at LF, at CRLF, or at a CR not followed by LF (the +/// three terminators the csv crate accepts), so every line ending convention +/// and any mix of them number lines alike. Offsets are asked for in +/// increasing order, so the count resumes where the last one stopped. +struct LineCounter<'a> { + body: &'a [u8], + /// Offset counted up to + pos: usize, + /// Line breaks in `body[..pos]` + breaks: usize, +} + +impl<'a> LineCounter<'a> { + fn new(body: &'a [u8]) -> Self { + LineCounter { + body, + pos: 0, + breaks: 0, } - i = end; } - (out, i >= data.len()) -} -/// The csv crate reports a comment line that ends the input WITHOUT a line -/// ending as an empty record. It is not a row: the comment is all there is. -fn is_phantom(record: &csv::ByteRecord) -> bool { - record.len() == 1 && record[0].is_empty() + /// The line `offset` is on. + fn line_at(&mut self, offset: usize) -> usize { + let offset = offset.min(self.body.len()); + if offset < self.pos { + self.pos = 0; + self.breaks = 0; + } + let body = self.body; + self.breaks += (self.pos..offset) + .filter(|&i| match body[i] { + b'\n' => true, + b'\r' => body.get(i + 1) != Some(&b'\n'), + _ => false, + }) + .count(); + self.pos = offset; + 1 + self.breaks + } } -/// Read the next record; false at end of input. `line` tracks the 1-based -/// line where the record starts, for error messages. +/// Read the next record; false at end of input. fn read_record( reader: &mut csv::Reader<&[u8]>, record: &mut csv::ByteRecord, source: &str, - line: &mut usize, + lines: &mut LineCounter, ) -> SqawkResult { - match reader.read_byte_record(record) { - Ok(more) => { - if let Some(pos) = record.position() { - *line = pos.line() as usize; - } - Ok(more) - } - Err(e) => Err(SqawkError::CsvParseError { + reader.read_byte_record(record).map_err(|e| { + let at = e.position().map_or(lines.pos, |p| p.byte() as usize); + SqawkError::CsvParseError { file: source.to_string(), - line: e.position().map_or(*line + 1, |p| p.line() as usize), + line: lines.line_at(at), error: e.to_string(), - }), + } + }) +} + +/// Where a writeback of `file_path` lands. +struct WriteTarget { + /// The file rewritten: `file_path` with symlinks resolved. + target: PathBuf, + /// The directory the temp file is created in. + parent: PathBuf, + /// How errors name the file: relative to the working directory when it + /// is inside it. Never the temp file. + shown: PathBuf, +} + +impl WriteTarget { + fn of(file_path: &Path) -> Self { + let target = match std::fs::canonicalize(file_path) { + Ok(real) => real, + // A file that does not exist yet (CREATE TABLE ... LOCATION). + Err(_) => file_path.to_path_buf(), + }; + let parent = match target.parent() { + Some(p) if !p.as_os_str().is_empty() => p.to_path_buf(), + _ => PathBuf::from("."), + }; + let shown = std::env::current_dir() + .and_then(std::fs::canonicalize) + .ok() + .and_then(|cwd| target.strip_prefix(cwd).ok().map(Path::to_path_buf)) + .unwrap_or_else(|| target.clone()); + WriteTarget { + target, + parent, + shown, + } + } +} + +/// Fail unless [`save_table`] can replace `file_path`, writing nothing: the +/// file must not be read-only (by its mode bits, or because opening it for +/// writing is refused), and its directory must take a new file (checked by +/// creating and removing an empty temp file there). A file that does not +/// exist yet passes when its directory does not exist either; `save_table` +/// creates it. +/// +/// The atomic rename would replace even a read-only file, so this check is +/// what keeps --write from overwriting one. +pub fn check_writable(file_path: &Path) -> SqawkResult<()> { + let WriteTarget { + target, + parent, + shown, + } = WriteTarget::of(file_path); + let read_only = || { + SqawkError::write( + &shown, + std::io::Error::new(std::io::ErrorKind::PermissionDenied, "file is read-only"), + ) + }; + if let Ok(meta) = std::fs::metadata(&target) { + if meta.permissions().readonly() { + return Err(read_only()); + } + if let Err(e) = std::fs::OpenOptions::new().append(true).open(&target) { + return Err(if e.kind() == std::io::ErrorKind::PermissionDenied { + read_only() + } else { + SqawkError::write(&shown, e) + }); + } + } + if parent.is_dir() { + NamedTempFile::new_in(&parent).map_err(|e| SqawkError::write(&shown, e))?; } + Ok(()) } /// Write `table` back to `file_path`, reproducing the source layout. @@ -317,269 +491,300 @@ fn read_record( /// synced, given the original file's permissions, and renamed over the /// target. A symlink is resolved first, so the link survives and its target /// is what gets rewritten. +/// +/// The rename replaces the directory entry, so a file with several hard +/// links is split from them: the other links keep the old contents. pub fn save_table(table: &Table, file_path: &Path) -> SqawkResult<()> { - let target = match std::fs::canonicalize(file_path) { - Ok(real) => real, - // A file that does not exist yet (CREATE TABLE ... LOCATION). - Err(_) => file_path.to_path_buf(), - }; - let parent = match target.parent() { - Some(p) if !p.as_os_str().is_empty() => p, - _ => Path::new("."), - }; - let temp_file = NamedTempFile::new_in(parent)?; + let WriteTarget { + target, + parent, + shown, + } = WriteTarget::of(file_path); + let fail = |e: std::io::Error| SqawkError::write(&shown, e); + let temp_file = NamedTempFile::new_in(&parent).map_err(fail)?; { let mut out = BufWriter::new(temp_file.as_file()); - table.write_source_layout(&mut out)?; - out.flush()?; + table.write_source_layout(&mut out).map_err(fail)?; + out.flush().map_err(fail)?; } if let Ok(meta) = std::fs::metadata(&target) { - temp_file.as_file().set_permissions(meta.permissions())?; + temp_file + .as_file() + .set_permissions(meta.permissions()) + .map_err(fail)?; } - temp_file.as_file().sync_all()?; - temp_file - .persist(&target) - .map_err(|e| SqawkError::IoError(e.error))?; + temp_file.as_file().sync_all().map_err(fail)?; + temp_file.persist(&target).map_err(|e| fail(e.error))?; Ok(()) } -/// Loads and saves CSV files (library convenience API). -#[allow(dead_code)] // not used by the binary; part of the library API -pub struct CsvHandler {} +#[cfg(test)] +mod tests { + use super::{read_table, unclosed_quote, ReadOptions}; + use crate::error::SqawkError; + use crate::table::Table; -impl Default for CsvHandler { - fn default() -> Self { - Self::new() + fn load(data: &[u8], opts: &ReadOptions) -> Result { + read_table("t", data.to_vec(), None, ",".into(), opts, "t.csv") + } + + fn written(t: &Table) -> Vec { + let mut out = Vec::new(); + t.write_source_layout(&mut out).unwrap(); + out } -} -#[allow(dead_code)] -impl CsvHandler { - /// Create a new CsvHandler - pub fn new() -> Self { - CsvHandler {} - } - - /// Save a table to a CSV file (see `save_table`) - pub fn save_csv(&self, table: &Table, file_path: &Path) -> SqawkResult<()> { - save_table(table, file_path) - } - - /// Load a CSV file into a table. - /// - /// * `file_spec` - `[table_name=]file_path` - /// * `custom_columns` - column names to use; the file then has no header - /// row and its first line is data - /// * `recover_errors` - skip malformed records instead of failing - /// - /// The first line is the header unless `custom_columns` is given. `#` - /// lines are data; use `load_with_options` for comment skipping. - pub fn load_csv( - &self, - file_spec: &str, - custom_columns: Option>, - recover_errors: Option, - ) -> SqawkResult
{ + #[test] + fn comments_are_skipped_and_kept() { + let data = std::fs::read("tests/data/commented.csv").unwrap(); let opts = ReadOptions { - columns: custom_columns, - recover_errors: recover_errors.unwrap_or(false), + comments: true, ..ReadOptions::default() }; - self.load_with_options(file_spec, &opts) - } - - /// Load a delimited file with explicit options. - pub fn load_with_options(&self, file_spec: &str, opts: &ReadOptions) -> SqawkResult
{ - let (table_name, file_path) = parse_file_spec(file_spec)?; - let data = std::fs::read(&file_path)?; - let delimiter = (opts.delimiter as char).to_string(); - read_table( - &table_name, - &data, - Some(file_path.clone()), - delimiter, - opts, - &file_path.to_string_lossy(), - ) + let table = load(&data, &opts).unwrap(); + assert_eq!(table.rows().len(), 3); + assert_eq!(table.rows()[0][1].to_string(), "Alice"); + assert_eq!(table.rows()[2][1].to_string(), "Charlie"); + assert_eq!(written(&table), data); } -} -/// Split `[table_name=]file_path`; the table name defaults to the file stem. -#[allow(dead_code)] -fn parse_file_spec(file_spec: &str) -> SqawkResult<(String, PathBuf)> { - if let Some((table_name, file_path)) = file_spec.split_once('=') { - return Ok((table_name.to_string(), PathBuf::from(file_path))); - } - let path = PathBuf::from(file_spec); - let stem = path - .file_stem() - .ok_or_else(|| { - SqawkError::InvalidFileSpec(format!("Invalid file specification: {}", file_spec)) - })? - .to_string_lossy() - .to_string(); - Ok((stem, path)) -} + #[test] + fn hash_lines_are_data_by_default() { + let table = load(b"#h\n#x\n1\n", &ReadOptions::default()).unwrap(); + assert_eq!(table.columns(), vec!["#h"]); + assert_eq!(table.rows().len(), 2); + assert_eq!(table.rows()[0][0].to_string(), "#x"); + } -#[cfg(test)] -mod tests { - use super::{CsvHandler, HeaderMode, ReadOptions}; - use crate::error::SqawkError; + #[test] + fn custom_columns_make_the_first_line_data() { + let opts = ReadOptions { + header: false, + columns: Some(vec!["x".into(), "y".into()]), + ..ReadOptions::default() + }; + let table = load(b"id,name\n1,a\n", &opts).unwrap(); + assert_eq!(table.columns(), vec!["x", "y"]); + assert_eq!(table.rows().len(), 2); + assert_eq!(table.rows()[0][1].to_string(), "name"); + assert_eq!(written(&table), b"id,name\n1,a\n"); + } - /// Options for a comma-separated file with `#` comment lines. Plain `.csv` - /// loading treats `#` lines as data; comment skipping is opt-in (the CLI - /// enables it for non-.csv delimited files). - fn commented_options(columns: Option>) -> ReadOptions { - ReadOptions { - delimiter: b',', - header: HeaderMode::Present, - comments: true, - columns, - recover_errors: false, + /// Columns given over a header: the header is skipped, must match the + /// column count, and is written back as read. + #[test] + fn custom_columns_over_a_header() { + let opts = ReadOptions { + columns: Some(vec!["x".into(), "y".into()]), + ..ReadOptions::default() + }; + let table = load(b"\"id\",name\r\n1,a\r\n", &opts).unwrap(); + assert_eq!(table.columns(), vec!["x", "y"]); + assert_eq!(table.rows().len(), 1); + assert_eq!(written(&table), b"\"id\",name\r\n1,a\r\n"); + match load(b"\n\na,b,c\n1,2,3\n", &opts) { + Err(SqawkError::CsvParseError { line, error, .. }) => { + assert_eq!(line, 3); + assert_eq!( + error, + "header has 3 fields, but the table declares 2 columns" + ); + } + other => panic!("unexpected: {other:?}"), } + // No header line yet: writeback gives the file one. + let table = load(b"", &opts).unwrap(); + assert_eq!(table.columns(), vec!["x", "y"]); + assert_eq!(written(&table), b"x,y\n"); } #[test] - fn test_csv_with_comments() { - // Create the CSV handler - let csv_handler = CsvHandler::new(); + fn too_many_fields_is_an_error() { + let data = std::fs::read("tests/data/malformed.csv").unwrap(); + match load(&data, &ReadOptions::default()) { + Err(SqawkError::CsvParseError { file, line, .. }) => { + assert_eq!(file, "t.csv"); + assert_eq!(line, 6); + } + other => panic!("unexpected: {other:?}"), + } + } - // Load the commented CSV file with comment skipping enabled - let table = csv_handler - .load_with_options("tests/data/commented.csv", &commented_options(None)) - .unwrap(); + /// The line a load error names. + fn error_line(data: &[u8], opts: &ReadOptions) -> usize { + match load(data, opts) { + Err(SqawkError::CsvParseError { line, .. }) => line, + other => panic!("{:?}: unexpected {other:?}", String::from_utf8_lossy(data)), + } + } - // Verify the correct data was loaded (comments should be ignored) - assert_eq!(table.rows().len(), 3); - assert_eq!(table.name(), "commented"); + /// Load errors must name the same line whatever the line endings. csv-core + /// ends a CRLF record at the CR and counts its LF only when the next + /// record is read, so its line numbers are one too low in CRLF files (a + /// Windows checkout of tests/data/malformed.csv reported line 5, not 6). + #[test] + fn load_error_line_is_the_same_for_crlf() { + let lf = b"id,name,age\n1,Alice,30\n3,Charlie,35\n5,Too,Many,Fields\n"; + let crlf = b"id,name,age\r\n1,Alice,30\r\n3,Charlie,35\r\n5,Too,Many,Fields\r\n"; + let quote = b"id,name\r\n1,\"a\r\nb\"\r\n2,x\r\n3,y,z\r\n"; + for (data, want) in [(&lf[..], 4), (&crlf[..], 4), ("e[..], 5)] { + assert_eq!( + error_line(data, &ReadOptions::default()), + want, + "{:?}", + String::from_utf8_lossy(data) + ); + } + } - // Check specific row data - let rows = table.rows(); - assert_eq!(rows[0][1].to_string(), "Alice"); - assert_eq!(rows[1][1].to_string(), "Bob"); - assert_eq!(rows[2][1].to_string(), "Charlie"); + /// Every line ending convention, any mix of them, quoted line breaks, + /// blank lines and a BOM: the error names the line the bad record is on. + #[test] + fn load_error_line_any_line_endings() { + let opts = ReadOptions::default(); + for (data, want) in [ + // Mixed endings. + (&b"a,b\r\n1,2\n3,4\r\n5,6,7\n"[..], 4), + (b"a,b\n1,2\r\n3,4\n5,6,7\r\n", 4), + // CR alone ends a record, so it ends a line. + (b"a,b\r1,2\r3,4,5\r", 3), + // Quoted fields holding LF, CRLF and both. + (b"a,b\n1,\"x\ny\"\n2,\"p\r\nq\"\r\n3,4,5\n", 6), + (b"a,b\r\n1,\"x\ny\r\nz\"\r\n3,4,5\r\n", 5), + // Blank lines (LF and CRLF) before the bad record. + (b"a,b\n\n1,2\n\n\n3,4,5\n", 6), + (b"a,b\r\n\r\n1,2\r\n\r\n\r\n3,4,5\r\n", 6), + // The BOM is not a line. + (b"\xef\xbb\xbfa,b\r\n1,2\r\n3,4,5\r\n", 3), + (b"\xef\xbb\xbfa,b\n1,2\n3,4,5\n", 3), + // The last record, without a line ending. + (b"a,b\r\n1,2\r\n3,4,5", 3), + ] { + assert_eq!( + error_line(data, &opts), + want, + "{:?}", + String::from_utf8_lossy(data) + ); + } } + /// Skipped `#` lines count as lines. #[test] - fn test_malformed_csv_with_error_recovery() { - // Create the CSV handler - let csv_handler = CsvHandler::new(); - - // Try to load the malformed CSV file with error recovery enabled - // We need to handle each row separately with flexible mode - let result = csv_handler.load_csv("tests/data/malformed.csv", None, Some(true)); - assert!(result.is_ok(), "Should succeed with error recovery enabled"); - let table = result.unwrap(); - - // In recovery mode we should get all rows that can be recovered - // The row with the unclosed quote will be skipped - // Row with missing field should be padded with a null - // Row with too many fields should be truncated - - // Check row count - assert!(table.rows().len() >= 3, "Should have at least 3 valid rows"); - - // Check specific row data for the rows we know should be there - let rows = table.rows(); - assert_eq!( - rows[0][1].to_string(), - "Alice", - "First row should have Alice" - ); - - // Due to the flexible recovery, exact row positions may vary based on implementation - // Instead, we'll check that certain expected values exist in the table - // Alice, Bob, and Charlie should all be present - - let mut has_alice = false; - let mut _has_bob = false; // Renamed to _has_bob since it's not asserted - let mut has_charlie = false; - - for row in table.rows() { - if row.len() > 1 { - let name = row[1].to_string(); - if name == "Alice" { - has_alice = true; - } else if name == "Bob with an unclosed quote\"" { - _has_bob = true; - } else if name == "Charlie" { - has_charlie = true; - } - } + fn load_error_line_after_comments() { + let opts = ReadOptions { + comments: true, + ..ReadOptions::default() + }; + for (data, want) in [ + (&b"# head\na,b\n# one\n1,2\n# two\n# three\n3,4,5\n"[..], 7), + ( + b"# head\r\na,b\r\n# one\r\n1,2\r\n# two\r\n# three\r\n3,4,5\r\n", + 7, + ), + (b"# head\r\na,b\n\r\n# one\r\n\n1,2\r\n3,4,5\n", 7), + (b"\xef\xbb\xbf# head\r\na,b\r\n3,4,5\r\n", 3), + ] { + assert_eq!( + error_line(data, &opts), + want, + "{:?}", + String::from_utf8_lossy(data) + ); + } + } + + /// An unclosed quote is reported at the line of its opening quote. + #[test] + fn unclosed_quote_line_any_line_endings() { + let opts = ReadOptions::default(); + for (data, want) in [ + (&b"a,b\n1,2\n3,\"x\n4,5\n"[..], 3), + (b"a,b\r\n1,2\r\n3,\"x\r\n4,5\r\n", 3), + (b"a,b\r\n1,\"p\r\nq\"\r\n\r\n3,\"x\r\n4,5\r\n", 5), + (b"\xef\xbb\xbfa,b\r\n1,2\n3,\"x\r\n", 3), + (b"a,b\r1,2\r3,\"x\r4,5\r", 3), + ] { + assert_eq!( + error_line(data, &opts), + want, + "{:?}", + String::from_utf8_lossy(data) + ); } + let comments = ReadOptions { + comments: true, + ..ReadOptions::default() + }; + assert_eq!(error_line(b"#c\r\na,b\r\n#d\r\n1,\"x\r\n", &comments), 4); + } - assert!(has_alice, "Table should contain Alice"); - assert!(has_charlie, "Table should contain Charlie"); + /// A bad header is reported at its own line, past BOM, blank and + /// comment lines. + #[test] + fn bad_header_line() { + assert_eq!(error_line(b"a,a\r\n1,2\r\n", &ReadOptions::default()), 1); + assert_eq!(error_line(b"\r\n\r\na,a\r\n", &ReadOptions::default()), 3); + let comments = ReadOptions { + comments: true, + ..ReadOptions::default() + }; + assert_eq!(error_line(b"\xef\xbb\xbf#c\r\n\na,,b\r\n", &comments), 3); } #[test] - fn test_malformed_csv_without_recovery() { - // Create the CSV handler - let csv_handler = CsvHandler::new(); - - // Try to load the malformed CSV file without error recovery - let result = csv_handler.load_csv("tests/data/malformed.csv", None, None); - - // Verify that we get an error - assert!(result.is_err()); - - // Check that the error contains CSV error information - // Type of error may vary by implementation (CsvParseError or InvalidSqlQuery) - let err = result.unwrap_err(); - match err { - SqawkError::CsvParseError { file, line, .. } => { - assert!(file.contains("malformed.csv")); - assert!(line > 0); - } - SqawkError::InvalidSqlQuery(msg) => { - assert!(msg.contains("columns")); - } - _ => panic!("Unexpected error type: {:?}", err), + fn untouched_bytes_round_trip() { + for data in [ + &b"k,s\n1,\"abc\"\n2,trail \n"[..], + b"\xef\xbb\xbfk,s\r\n1,a\r\n\r\n2,\"x\r\ny\"\n3,\xff", + b"k,s,t\n1\n\n\n2,b,c", + b"a\n", + b"a", + b"", + b"\n\n", + b"a,b\r1,2\r3,4\r", + ] { + let table = load(data, &ReadOptions::default()); + let table = match table { + Ok(t) => t, + Err(_) => continue, + }; + assert_eq!(written(&table), data, "{:?}", String::from_utf8_lossy(data)); + } + let opts = ReadOptions { + comments: true, + ..ReadOptions::default() + }; + for data in [&b"#c\na\n#d\n1\n#e"[..], b"#only", b"a\n1\n# tail\n\n"] { + let table = load(data, &opts).unwrap(); + assert_eq!(written(&table), data, "{:?}", String::from_utf8_lossy(data)); } } #[test] - fn test_csv_with_custom_columns() { - // Create the CSV handler - let csv_handler = CsvHandler::new(); - - // Define custom column names - let custom_columns = Some(vec![ - "user_id".to_string(), - "full_name".to_string(), - "user_age".to_string(), - ]); - - // Load the commented CSV file with custom columns - let table = csv_handler - .load_with_options( - "tests/data/commented.csv", - &commented_options(custom_columns), - ) - .unwrap(); - - // Verify the column names were properly set - let columns = table.columns(); - assert_eq!(columns[0], "user_id"); - assert_eq!(columns[1], "full_name"); - assert_eq!(columns[2], "user_age"); - - // Custom columns mean the file has no header row, so its first - // non-comment line ("id,name,age") is data: 4 rows, not 3. - assert_eq!(table.rows().len(), 4); - assert_eq!(table.rows()[0][1].to_string(), "name"); - assert_eq!(table.rows()[1][1].to_string(), "Alice"); + fn unclosed_quote_detection() { + assert_eq!(unclosed_quote(b"1,\"2\n3,4\n", b','), Some(2)); + assert_eq!(unclosed_quote(b"1,\"2\"\n", b','), None); + assert_eq!(unclosed_quote(b"1,\"a\"\"b\"\n", b','), None); + assert_eq!(unclosed_quote(b"1,\"a\"\"\n", b','), Some(2)); + assert_eq!(unclosed_quote(b"1,a\"b\n", b','), None); + assert_eq!(unclosed_quote(b"\"a\"x\"y", b','), None); + assert!(matches!( + load(b"a,b\n1,\"2\n3,4\n", &ReadOptions::default()), + Err(SqawkError::CsvParseError { line: 2, .. }) + )); } #[test] - fn test_csv_hash_lines_are_data_by_default() { - // Without comment skipping, the first line is the header even though it - // starts with '#', and '#' lines are ordinary records. - let table = CsvHandler::new() - .load_csv("tests/data/commented.csv", None, Some(true)) - .unwrap(); - assert_eq!(table.columns(), vec!["# This is a header comment"]); + fn bad_header_names() { + for data in [&b"a,a\n"[..], b"a,A\n", b"a,,c\n", b",\n"] { + assert!( + load(data, &ReadOptions::default()).is_err(), + "{:?}", + String::from_utf8_lossy(data) + ); + } } } diff --git a/src/database.rs b/src/database.rs index 2bb8b03..d3f9ad7 100644 --- a/src/database.rs +++ b/src/database.rs @@ -12,28 +12,179 @@ use crate::capacity::DEFAULT_TABLE_CAPACITY; use crate::config::AppConfig; use crate::error::{SqawkError, SqawkResult}; use crate::table::Table; -use std::collections::HashMap; +use crate::vm::bytecode::Program; +use std::cell::OnceCell; +use std::collections::{HashMap, HashSet}; /// Represents the central database that holds all tables pub struct Database { /// In-memory tables indexed by their names tables: HashMap, + + /// The tables of `tables` whose rows a program computes when a cursor + /// first opens them (see [`Database::add_lazy_table`]). + lazy: HashMap, + + /// Names of the tables whose contents or schema changed since they were + /// loaded or created: the tables `--write` writes back. DROP TABLE takes + /// a name out again. + modified: HashSet, + + /// The open transaction, if any (BEGIN ... COMMIT / ROLLBACK). + transaction: Option, +} + +/// What ROLLBACK needs to put the database back as it was at BEGIN. +/// +/// Copy-on-write: a table is copied the first time the transaction is about +/// to change it (`get_table_mut`), moved here when the transaction drops it, +/// and recorded as absent when the transaction creates it. BEGIN itself +/// copies nothing, and a table the transaction only reads is never copied. +struct Transaction { + /// Each table the transaction touched, as it was at BEGIN (`None`: it + /// did not exist). + originals: HashMap>, + /// The modified-table set at BEGIN. + modified: HashSet, +} + +/// A CTE's or derived table's rows, computed on first use: SQLite codes such +/// a subquery behind OP_Once (or as a co-routine) where the query first +/// reads it, so a subquery that is never read is never run. +pub struct LazyTable { + /// The query; its result columns are the table's. + program: Program, + /// The table with its rows, once computed. + filled: OnceCell
, +} + +impl LazyTable { + /// The program that computes the rows. + pub fn program(&self) -> &Program { + &self.program + } + + /// The computed table, or `None` before it is computed. + pub fn filled(&self) -> Option<&Table> { + self.filled.get() + } + + /// Record the computed table and return it. A table already recorded + /// is kept: its rows are the same. + pub fn fill(&self, table: Table) -> &Table { + self.filled.get_or_init(|| table) + } } /// Methods for direct table manipulation - used only in special cases impl Database { + /// Add `table`, which has columns and no rows, as a table whose rows + /// `program` computes when a cursor first opens it ([`LazyTable`]). The + /// program runs over the database as it is then; nothing it reads + /// changes within a statement, and the table lives only for one. + pub fn add_lazy_table( + &mut self, + name: String, + table: Table, + program: Program, + ) -> SqawkResult<()> { + self.add_table(name.clone(), table)?; + self.lazy.insert( + name, + LazyTable { + program, + filled: OnceCell::new(), + }, + ); + Ok(()) + } + + /// The lazy table named `name`, if it is one. + pub fn lazy_table(&self, name: &str) -> Option<&LazyTable> { + self.lazy.get(name) + } + /// Remove a table from the database by name /// - /// This is a specialized method intended for use when replacing tables - /// that were loaded with different schemas. - /// /// # Arguments /// * `name` - The name of the table to remove /// /// # Returns /// * `true` if the table was found and removed, `false` otherwise pub fn remove_table(&mut self, name: &str) -> bool { - self.tables.remove(name).is_some() + let Some(table) = self.tables.remove(name) else { + return false; + }; + self.lazy.remove(name); + self.modified.remove(name); + if let Some(txn) = &mut self.transaction { + txn.originals.entry(name.to_string()).or_insert(Some(table)); + } + true + } +} + +/// Transactions and the modified-table set. +impl Database { + /// BEGIN: open a transaction. + pub fn begin(&mut self) -> SqawkResult<()> { + if self.transaction.is_some() { + return Err(SqawkError::InvalidSqlQuery( + "cannot start a transaction within a transaction".to_string(), + )); + } + self.transaction = Some(Transaction { + originals: HashMap::new(), + modified: self.modified.clone(), + }); + Ok(()) + } + + /// COMMIT / END: keep the transaction's changes. + pub fn commit(&mut self) -> SqawkResult<()> { + self.transaction.take().map(drop).ok_or_else(|| { + SqawkError::InvalidSqlQuery("cannot commit - no transaction is active".to_string()) + }) + } + + /// ROLLBACK: put every table, and the modified-table set, back as they + /// were at BEGIN. + pub fn rollback(&mut self) -> SqawkResult<()> { + let txn = self.transaction.take().ok_or_else(|| { + SqawkError::InvalidSqlQuery("cannot rollback - no transaction is active".to_string()) + })?; + for (name, original) in txn.originals { + self.tables.remove(&name); + if let Some(table) = original { + self.tables.insert(name, table); + } + } + self.modified = txn.modified; + Ok(()) + } + + /// Whether a transaction is open. + pub fn in_transaction(&self) -> bool { + self.transaction.is_some() + } + + /// Record that `name` changed and must be written back by `--write`. + pub fn mark_modified(&mut self, name: &str) { + if !self.modified.contains(name) { + self.modified.insert(name.to_string()); + } + } + + /// Whether `name` changed since it was loaded or created. + pub fn is_modified(&self, name: &str) -> bool { + self.modified.contains(name) + } + + /// The modified tables, in ascending name order. + pub fn modified_tables(&self) -> Vec { + let mut names: Vec = self.modified.iter().cloned().collect(); + names.sort(); + names } } @@ -48,6 +199,9 @@ impl Database { pub fn new() -> Self { Database { tables: HashMap::with_capacity(DEFAULT_TABLE_CAPACITY), + lazy: HashMap::new(), + modified: HashSet::new(), + transaction: None, } } @@ -65,6 +219,9 @@ impl Database { return Err(SqawkError::TableAlreadyExists(name)); } + if let Some(txn) = &mut self.transaction { + txn.originals.entry(name.clone()).or_insert(None); + } self.tables.insert(name, table); Ok(()) } @@ -91,7 +248,17 @@ impl Database { /// # Returns /// * `Ok(&mut Table)` mutable reference to the requested table /// * `Err` if the table doesn't exist + /// + /// Inside a transaction, the first call for a table saves a copy of it + /// for ROLLBACK. pub fn get_table_mut(&mut self, name: &str) -> SqawkResult<&mut Table> { + if let Some(txn) = &mut self.transaction { + if !txn.originals.contains_key(name) { + if let Some(table) = self.tables.get(name) { + txn.originals.insert(name.to_string(), Some(table.clone())); + } + } + } self.tables .get_mut(name) .ok_or_else(|| SqawkError::TableNotFound(name.to_string())) @@ -141,50 +308,74 @@ impl Database { /// # Arguments /// * `config` - The application configuration containing table definitions /// + /// Each definition is `NAME:COL1,COL2,...` and becomes a placeholder + /// table that the input file loaded as NAME fills in (see + /// [`Database::check_table_definitions_used`]). + /// /// # Returns /// * `Ok(())` if all table definitions were compiled successfully - /// * `Err` if there was an issue with any table definition + /// * `Err` for a definition without `:`, an empty table or column name, + /// a column named twice, or a table defined twice pub fn compile_table_definitions(&mut self, config: &AppConfig) -> SqawkResult<()> { - // Process each table definition string (format: "table_name:col1,col2,...") for tabledef in config.table_definitions() { - if let Some((table_name, columns_str)) = tabledef.split_once(':') { - let columns = columns_str - .split(',') - .map(|s| s.trim().to_string()) - .collect::>(); - - if !columns.is_empty() { - // Create a new table with the specified columns - // Note: No file_path is assigned here as this is a table definition only - let columns_len = columns.len(); // Store the length before we move columns - let mut table = Table::new(table_name, columns, None); - table.set_placeholder(true); - - // Add the table to the database, overwriting any existing definition with the same name - // This ensures CLI definitions take precedence - if self.has_table(table_name) { - // Remove the existing table first to avoid conflict errors - // Use direct replacement via insert/remove to avoid errors with private fields - self.tables.remove(table_name); - } - - self.add_table(table_name.to_string(), table)?; - - if config.verbose() { - eprintln!( - "Compiled table definition for '{}' with {} columns", - table_name, columns_len - ); - } + let bad = |why: &str| { + SqawkError::InvalidFileSpec(format!("invalid --tabledef '{}': {}", tabledef, why)) + }; + let (table_name, columns_str) = tabledef + .split_once(':') + .ok_or_else(|| bad("expected NAME:COL1,COL2,..."))?; + let table_name = table_name.trim(); + if table_name.is_empty() { + return Err(bad("empty table name")); + } + let columns: Vec = columns_str + .split(',') + .map(|s| s.trim().to_string()) + .collect(); + for (i, col) in columns.iter().enumerate() { + if col.is_empty() { + return Err(bad("empty column name")); + } + if columns[..i].iter().any(|c| c.eq_ignore_ascii_case(col)) { + return Err(bad(&format!("duplicate column name: {}", col))); } - } else if config.verbose() { - eprintln!("Invalid table definition format: {}", tabledef); - eprintln!("Expected format: table_name:col1,col2,..."); + } + if self.has_table(table_name) { + return Err(bad(&format!("table {} is defined twice", table_name))); + } + + let columns_len = columns.len(); + let mut table = Table::new(table_name, columns, None); + table.set_placeholder(true); + self.add_table(table_name.to_string(), table)?; + + if config.verbose() { + diag!( + "Compiled table definition for '{}' with {} columns", + table_name, + columns_len + ); } } Ok(()) } + + /// Fail if a `--tabledef` names a table that no input file loaded: its + /// placeholder is still in the database once every file is loaded. + pub fn check_table_definitions_used(&self) -> SqawkResult<()> { + match self + .table_names() + .into_iter() + .find(|name| self.tables[name].is_placeholder()) + { + Some(name) => Err(SqawkError::InvalidFileSpec(format!( + "--tabledef names table '{}', which no input file loads", + name + ))), + None => Ok(()), + } + } } #[cfg(test)] @@ -228,7 +419,7 @@ mod tests { assert!(db.has_table("table2")); // Remove a table manually - db.tables.remove("table1"); + db.remove_table("table1"); assert_eq!(db.table_count(), 1); assert!(!db.has_table("table1")); assert!(db.has_table("table2")); diff --git a/src/diag.rs b/src/diag.rs new file mode 100644 index 0000000..ea3f252 --- /dev/null +++ b/src/diag.rs @@ -0,0 +1,42 @@ +//! Diagnostic output on standard error. +//! +//! Every message sqawk writes to stderr -- error lines and `-v` debugging +//! output alike -- goes through [`write_line`]. A failed write is ignored: +//! a closed or full stderr (`sqawk -v ... 2>&1 | head`) must never abort a +//! run, which `eprintln!` does by panicking. + +use std::io::Write; + +/// Write one line to standard error, ignoring any write error. +pub fn write_line(args: std::fmt::Arguments<'_>) { + let stderr = std::io::stderr(); + let mut err = stderr.lock(); + let _ = err.write_fmt(args).and_then(|()| err.write_all(b"\n")); +} + +/// Write one line to standard output, ignoring any write error (a reader +/// that has gone away is not an error worth aborting for). +pub fn out_line(args: std::fmt::Arguments<'_>) { + let stdout = std::io::stdout(); + let mut out = stdout.lock(); + let _ = out.write_fmt(args).and_then(|()| out.write_all(b"\n")); +} + +/// `eprintln!` that never panics: formats a line onto stderr and ignores +/// write errors. +macro_rules! diag { + ($($arg:tt)*) => { + $crate::diag::write_line(format_args!($($arg)*)) + }; +} + +/// `println!` that never panics: formats a line onto stdout and ignores +/// write errors. +macro_rules! outln { + () => { + $crate::diag::out_line(format_args!("")) + }; + ($($arg:tt)*) => { + $crate::diag::out_line(format_args!($($arg)*)) + }; +} diff --git a/src/error.rs b/src/error.rs index ac72add..3b69dd6 100644 --- a/src/error.rs +++ b/src/error.rs @@ -4,9 +4,15 @@ //! It provides a comprehensive error handling system that categorizes //! different failure modes, supports error propagation, and supplies //! helpful error messages to users. +//! +//! Every error's `Display` is one complete, self-contained line: the CLI +//! prints it as `sqawk: `. No variant exposes an inner error through +//! `source()`, because its message is already part of the line; a caller +//! that walked the chain would print the cause twice. use std::error::Error; use std::fmt; +use std::path::Path; /// SqawkError represents all possible errors that can occur in the sqawk application /// @@ -24,6 +30,14 @@ pub enum SqawkError { /// Error during file system operations (reading/writing files) IoError(std::io::Error), + /// An I/O error on a named file: ` '': `, e.g. + /// `cannot open 'x.csv': Permission denied` + FileError { + action: &'static str, + path: String, + error: std::io::Error, + }, + /// Error while parsing or processing delimited file data CsvError(csv::Error), @@ -43,9 +57,6 @@ pub enum SqawkError { /// Error when trying to create a table that already exists TableAlreadyExists(String), - /// Error when a file doesn't exist - FileNotFound(String), - /// Error when a table doesn't have an associated file path NoFilePath(String), @@ -55,7 +66,7 @@ pub enum SqawkError { /// Error for invalid file=table specifications InvalidFileSpec(String), - /// Error for SQL features that aren't implemented yet + /// Error for SQL that sqawk parses but does not implement UnsupportedSqlFeature(String), /// Error for semantically invalid SQL queries @@ -65,42 +76,114 @@ pub enum SqawkError { VmError(String), } +impl SqawkError { + /// An I/O error opening (reading) the file at `path`. + pub fn open(path: impl AsRef, error: std::io::Error) -> Self { + SqawkError::FileError { + action: "cannot open", + path: display_path(path.as_ref()), + error, + } + } + + /// An I/O error writing the file at `path`. + pub fn write(path: impl AsRef, error: std::io::Error) -> Self { + SqawkError::FileError { + action: "cannot write", + path: display_path(path.as_ref()), + error, + } + } +} + +/// `path` as shown to the user. A canonical Windows path is verbatim +/// (`\\?\C:\dir\f.csv`, `\\?\UNC\server\share\f.csv`), a spelling +/// the user never types; it is shown as the plain path (`C:\dir\f.csv`, +/// `\\server\share\f.csv`). The canonical form stays the file's identity. +pub fn display_path(path: &Path) -> String { + let shown = path.display().to_string(); + if cfg!(windows) { + strip_verbatim(&shown).to_string() + } else { + shown + } +} + +/// `s` without a Windows verbatim prefix: `\\?\UNC\` becomes `\\`, and a +/// `\\?\` before a drive letter is dropped. Any other verbatim path (a +/// volume GUID, say) has no plain spelling and is kept. +fn strip_verbatim(s: &str) -> std::borrow::Cow<'_, str> { + if let Some(rest) = s.strip_prefix(r"\\?\UNC\") { + return format!(r"\\{rest}").into(); + } + match s.strip_prefix(r"\\?\") { + Some(rest) if is_drive_path(rest) => rest.into(), + _ => s.into(), + } +} + +/// Whether `s` starts with a drive letter and a colon (`C:`). +fn is_drive_path(s: &str) -> bool { + let b = s.as_bytes(); + b.len() >= 2 && b[0].is_ascii_alphabetic() && b[1] == b':' +} + +/// The reason part of an I/O error, without the `(os error N)` suffix the +/// standard library appends, and without the `at path "..."` suffix the +/// tempfile crate appends (which names an internal temporary file). +pub fn io_reason(e: &std::io::Error) -> String { + let mut s = e.to_string(); + if let Some(i) = s.find(" at path \"") { + s.truncate(i); + } + if let Some(i) = s.find(" (os error ") { + s.truncate(i); + } + s +} + impl fmt::Display for SqawkError { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { match self { - SqawkError::IoError(e) => write!(f, "I/O error: {}", e), - SqawkError::CsvError(e) => write!(f, "File parsing error: {}", e), + SqawkError::IoError(e) => write!(f, "{}", io_reason(e)), + SqawkError::FileError { + action, + path, + error, + } => write!(f, "{} '{}': {}", action, path, io_reason(error)), + SqawkError::CsvError(e) => write!(f, "{}", e), SqawkError::CsvParseError { file, line, error } => { - write!(f, "CSV parse error in {} at line {}: {}", file, line, error) + write!(f, "{}: line {}: {}", file, line, error) } - SqawkError::SqlParseError(e) => write!(f, "SQL parsing error: {}", e), - SqawkError::TableNotFound(name) => write!(f, "Table '{}' not found", name), - SqawkError::TableAlreadyExists(name) => write!(f, "Table '{}' already exists", name), - SqawkError::FileNotFound(path) => write!(f, "File not found: {}", path), - SqawkError::NoFilePath(name) => { - write!(f, "Table '{}' has no associated file path", name) + SqawkError::SqlParseError(e) => { + use sqlparser::parser::ParserError; + match e { + ParserError::TokenizerError(s) | ParserError::ParserError(s) => { + write!(f, "{}", s) + } + ParserError::RecursionLimitExceeded => write!(f, "recursion limit exceeded"), + } } - SqawkError::ColumnNotFound(name) => write!(f, "Column '{}' not found", name), - SqawkError::InvalidFileSpec(spec) => write!(f, "Invalid file specification: {}", spec), - SqawkError::UnsupportedSqlFeature(feature) => { - write!(f, "Unsupported SQL feature: {}", feature) + SqawkError::TableNotFound(name) => write!(f, "no such table: {}", name), + SqawkError::TableAlreadyExists(name) => write!(f, "table {} already exists", name), + SqawkError::NoFilePath(name) => { + write!( + f, + "table '{}' has no associated file path \ + (CREATE TABLE ... LOCATION 'file.csv' gives it one)", + name + ) } - SqawkError::InvalidSqlQuery(msg) => write!(f, "Invalid SQL query: {}", msg), - SqawkError::VmError(msg) => write!(f, "VM execution error: {}", msg), + SqawkError::ColumnNotFound(name) => write!(f, "no such column: {}", name), + SqawkError::InvalidFileSpec(msg) + | SqawkError::UnsupportedSqlFeature(msg) + | SqawkError::InvalidSqlQuery(msg) + | SqawkError::VmError(msg) => write!(f, "{}", msg), } } } -impl Error for SqawkError { - fn source(&self) -> Option<&(dyn Error + 'static)> { - match self { - SqawkError::IoError(e) => Some(e), - SqawkError::CsvError(e) => Some(e), - SqawkError::SqlParseError(e) => Some(e), - _ => None, - } - } -} +impl Error for SqawkError {} // From implementations for automatic error conversion (replaces #[from]) @@ -128,24 +211,7 @@ impl From for SqawkError { // but don't care about the exact error message impl PartialEq for SqawkError { fn eq(&self, other: &Self) -> bool { - // Match on self and other to check if they are the same variant - match (self, other) { - (SqawkError::IoError(_), SqawkError::IoError(_)) => true, - (SqawkError::CsvError(_), SqawkError::CsvError(_)) => true, - (SqawkError::CsvParseError { .. }, SqawkError::CsvParseError { .. }) => true, - (SqawkError::SqlParseError(_), SqawkError::SqlParseError(_)) => true, - (SqawkError::TableNotFound(_), SqawkError::TableNotFound(_)) => true, - (SqawkError::TableAlreadyExists(_), SqawkError::TableAlreadyExists(_)) => true, - (SqawkError::FileNotFound(_), SqawkError::FileNotFound(_)) => true, - (SqawkError::NoFilePath(_), SqawkError::NoFilePath(_)) => true, - (SqawkError::ColumnNotFound(_), SqawkError::ColumnNotFound(_)) => true, - (SqawkError::InvalidFileSpec(_), SqawkError::InvalidFileSpec(_)) => true, - (SqawkError::UnsupportedSqlFeature(_), SqawkError::UnsupportedSqlFeature(_)) => true, - (SqawkError::InvalidSqlQuery(_), SqawkError::InvalidSqlQuery(_)) => true, - (SqawkError::VmError(_), SqawkError::VmError(_)) => true, - // If variants are different, they are not equal - _ => false, - } + std::mem::discriminant(self) == std::mem::discriminant(other) } } @@ -154,3 +220,62 @@ impl PartialEq for SqawkError { /// This type alias simplifies function signatures and error handling throughout the codebase. /// It represents either a successful result of type `T` or a `SqawkError`. pub type SqawkResult = std::result::Result; + +#[cfg(test)] +mod tests { + use super::*; + + /// The platform's "permission denied" OS error. Raw error numbers differ + /// by platform: 13 is EACCES on Unix but ERROR_INVALID_DATA on Windows, + /// where access denied is 5. + #[cfg(unix)] + const DENIED: i32 = 13; + #[cfg(windows)] + const DENIED: i32 = 5; + + /// The OS's own text for `DENIED` without the "(os error N)" suffix. The + /// wording comes from the OS (and differs between Windows and Wine), so + /// it is read rather than spelled out. + fn denied_text() -> String { + let shown = std::io::Error::from_raw_os_error(DENIED).to_string(); + let suffix = format!(" (os error {})", DENIED); + let text = shown + .strip_suffix(&suffix) + .expect("io::Error display ends in (os error N)"); + text.to_string() + } + + #[test] + fn io_reason_strips_os_error_and_temp_path() { + let e = std::io::Error::from_raw_os_error(DENIED); + assert_eq!(io_reason(&e), denied_text()); + // glibc's wording; Miri supplies its own lower-case text. + #[cfg(all(unix, not(miri)))] + assert_eq!(io_reason(&e), "Permission denied"); + let e = std::io::Error::other("Permission denied (os error 13) at path \"/d/.tmpAb12\""); + assert_eq!(io_reason(&e), "Permission denied"); + } + + #[test] + fn file_error_names_file_once() { + let e = SqawkError::open("np.csv", std::io::Error::from_raw_os_error(DENIED)); + assert_eq!( + e.to_string(), + format!("cannot open 'np.csv': {}", denied_text()) + ); + } + + #[test] + fn verbatim_prefix_is_not_shown() { + assert_eq!(strip_verbatim(r"\\?\C:\d\f.csv"), r"C:\d\f.csv"); + assert_eq!( + strip_verbatim(r"\\?\UNC\srv\share\f.csv"), + r"\\srv\share\f.csv" + ); + assert_eq!( + strip_verbatim(r"\\?\Volume{x}\f.csv"), + r"\\?\Volume{x}\f.csv" + ); + assert_eq!(strip_verbatim("d/f.csv"), "d/f.csv"); + } +} diff --git a/src/file_handler.rs b/src/file_handler.rs index b1daf1e..22bba39 100644 --- a/src/file_handler.rs +++ b/src/file_handler.rs @@ -10,10 +10,10 @@ use std::path::{Path, PathBuf}; use crate::config::AppConfig; -use crate::csv_handler::{self, HeaderMode, ReadOptions}; +use crate::csv_handler::{self, ReadOptions}; use crate::database::Database; use crate::error::{SqawkError, SqawkResult}; -use crate::table::{delimiter_byte, Table}; +use crate::table::{delimiter_byte, ColumnDefinition, FileLayout, Table}; /// File operand that names standard input rather than a path on disk. pub const STDIN_SPEC: &str = "-"; @@ -21,15 +21,6 @@ pub const STDIN_SPEC: &str = "-"; /// Table name given to a bare `-` operand. pub const STDIN_TABLE: &str = "stdin"; -/// Enum representing different file formats supported by sqawk -#[derive(Debug, Clone, Copy)] -pub enum FileFormat { - /// CSV (comma-separated values) - Csv, - /// Delimiter-separated values - Delimited, -} - /// Unified file handler that delegates to specific format handlers /// /// This struct provides a consistent interface for working with different file formats @@ -39,13 +30,11 @@ pub enum FileFormat { /// Key responsibilities: /// - Determining the appropriate handler based on file format /// - Managing the relationship between tables and their source files -/// - Providing access to the database for table operations /// - Handling table loading and saving with proper format detection +/// +/// It holds no tables: the [`Database`] it loads into or saves from is +/// passed to each call. pub struct FileHandler { - /// Reference to a database object which is the source of truth for tables - /// Stored as a raw pointer to avoid borrowing limitations - database: *mut Database, - /// Application configuration for global settings like verbosity and field separators config: AppConfig, @@ -62,23 +51,16 @@ pub struct FileHandler { stdin_available: bool, } -// Add safety implementation for the raw pointer to Database -unsafe impl Send for FileHandler {} -unsafe impl Sync for FileHandler {} - impl FileHandler { - /// Create a new FileHandler with application config and database + /// Create a new FileHandler with application config /// /// # Arguments /// * `config` - Application configuration with global settings - /// * `database` - Mutable reference to the database to use as source of truth /// /// # Returns /// A new FileHandler instance ready to load and manage tables - pub fn new(config: &AppConfig, database: &mut Database) -> Self { + pub fn new(config: &AppConfig) -> Self { FileHandler { - // SAFETY: The caller must ensure that the database outlives this FileHandler - database: database as *mut Database, config: config.clone(), stdin_consumed: false, stdin_table: None, @@ -112,129 +94,99 @@ impl FileHandler { } self.stdin_consumed = true; let mut data = Vec::new(); - std::io::Read::read_to_end(&mut std::io::stdin().lock(), &mut data)?; + std::io::Read::read_to_end(&mut std::io::stdin().lock(), &mut data) + .map_err(|e| SqawkError::open("standard input", e))?; Ok(data) } - /// Get a mutable reference to the database + /// Load a file into an in-memory table and add it to `db` /// - /// Provides safe access to the database reference stored as a raw pointer. - /// This design allows the FileHandler to maintain a reference to the Database - /// while avoiding Rust's borrowing conflicts in complex operations. + /// * `file_spec` - `[table_name=]file_path` (see [`parse_table_spec`]); + /// `-` is standard input /// - /// # Safety - /// Safety is guaranteed because: - /// - The database pointer is initialized in the constructor and never changes - /// - The FileHandler's lifetime is tied to the Database through the constructor's contract - /// - All access to the database happens through this controlled interface + /// The delimiter is -F if given, else tab for `.tsv`/`.tab` files and + /// comma for everything else. The first line is the header unless + /// `--no-header` or `--tabledef` supplies the column names. `#` lines + /// are comments only under `--comments`. /// - /// # Returns - /// * `&mut Database` - Mutable reference to the database - fn database_mut(&mut self) -> &mut Database { - // SAFETY: The caller of `new` ensures the database outlives this FileHandler, - // and we have exclusive access through `&mut self` - unsafe { &mut *self.database } - } - - /// Load a file into an in-memory table and add it to the database - /// - /// This method handles the complete file loading process: - /// 1. Parses the file specification to extract table name and file path - /// 2. Checks if the table already exists in the database - /// 3. Automatically detects file format (CSV or delimiter-separated) - /// 4. Delegates to the appropriate handler based on format - /// 5. Adds the loaded table to the database - /// - /// If a table with the same name already exists in the database, it will be - /// replaced with the newly loaded table. - /// - /// # Arguments - /// * `file_spec` - File specification in format [table_name=]file_path + /// The table records the file's canonical absolute path, so a later + /// `.cd` in the REPL does not change which file `.save` writes, and the + /// same file cannot be loaded twice under different names (writeback + /// would make one copy overwrite the other). /// /// # Returns - /// * `SqawkResult>` - Tuple of (table_name, file_path) if successful - pub fn load_file(&mut self, file_spec: &str) -> SqawkResult> { - let (table_name, file_path) = self.parse_file_spec(file_spec)?; + /// * `(table_name, file_path)` as given + pub fn load_file( + &mut self, + db: &mut Database, + file_spec: &str, + ) -> SqawkResult> { + let (table_name, file_path) = parse_table_spec(file_spec)?; // A "-" operand reads standard input. It is parsed as CSV (comma, // header row) unless -F says otherwise, which matches what sqawk // itself writes to stdout, so `sqawk ... | sqawk ... -` round-trips. // It has no writeback target: --write must never touch it. let from_stdin = file_path == Path::new(STDIN_SPEC); - let file_path_str = if from_stdin { - STDIN_SPEC.to_string() + let file_path_str = file_path.to_string_lossy().to_string(); + + let canonical = if from_stdin { + None } else { - file_path.to_string_lossy().to_string() + let real = + std::fs::canonicalize(&file_path).map_err(|e| SqawkError::open(&file_path, e))?; + if real.is_dir() { + return Err(SqawkError::open( + &file_path, + std::io::Error::new(std::io::ErrorKind::IsADirectory, "Is a directory"), + )); + } + Some(real) }; // A table that already exists is either a --tabledef placeholder, // whose column names this file supplies the data for, or a genuine // table, in which case two inputs resolve to the same name. - let predefined_columns: Option> = { - let db = self.database_mut(); - match db.get_table(&table_name) { - Ok(t) if t.is_placeholder() => Some(t.columns().to_vec()), - Ok(_) => return Err(SqawkError::TableAlreadyExists(table_name)), - Err(_) => None, - } + let predefined_columns: Option> = match db.get_table(&table_name) { + Ok(t) if t.is_placeholder() => Some(t.columns().to_vec()), + Ok(_) => return Err(SqawkError::TableAlreadyExists(table_name)), + Err(_) => None, }; + if let Some(real) = &canonical { + check_not_loaded(db, real, &file_path_str)?; + } let verbose = self.config.verbose(); if predefined_columns.is_some() && verbose { - eprintln!( + diag!( "Table '{}' has predefined schema, using those column names", table_name ); } - // Delimiter: -F if given, else comma for .csv (and stdin and files - // without an extension), tab for anything else. - let format = if from_stdin { - FileFormat::Csv - } else { - self.detect_format(&file_path) - }; - let delimiter_str = self - .config - .field_separator() - .unwrap_or_else(|| match format { - FileFormat::Csv => ",".to_string(), - FileFormat::Delimited => "\t".to_string(), - }); - - // Header policy: a .csv file (or stdin) always has a header row, and - // its '#' lines are data. Other delimited files get the headerless - // guess (for files like /etc/passwd) and '#' comment lines. - let csv_rules = from_stdin || Self::has_csv_extension(&file_path); - let header = if self.config.no_header() { - HeaderMode::Absent - } else if csv_rules { - HeaderMode::Present - } else { - HeaderMode::Guess - }; + let delimiter_str = self.config.field_separator().unwrap_or_else(|| { + if !from_stdin && has_tab_extension(&file_path) { + "\t".to_string() + } else { + ",".to_string() + } + }); let opts = ReadOptions { delimiter: delimiter_byte(&delimiter_str), - header, - comments: !csv_rules, + // --tabledef names the columns; the first line is data. + header: !self.config.no_header() && predefined_columns.is_none(), + comments: self.config.comments(), columns: predefined_columns.clone(), - recover_errors: false, }; - let data = if from_stdin { - self.read_stdin()? - } else { - std::fs::read(&file_path)? - }; - let target = if from_stdin { - None - } else { - Some(file_path.clone()) + let data = match &canonical { + None => self.read_stdin()?, + Some(real) => std::fs::read(real).map_err(|e| SqawkError::open(&file_path, e))?, }; let table = csv_handler::read_table( &table_name, - &data, - target, + data, + canonical, delimiter_str, &opts, &file_path_str, @@ -254,11 +206,10 @@ impl FileHandler { self.stdin_table = Some(table_name.clone()); } - let db = self.database_mut(); if predefined_columns.is_some() { // Replace the --tabledef placeholder with the loaded table. if verbose { - eprintln!( + diag!( "Replacing predefined schema for '{}' with loaded data", table_name ); @@ -270,96 +221,6 @@ impl FileHandler { Ok(Some((table_name, file_path_str))) } - /// Whether `path` has a `.csv` extension (any case). - fn has_csv_extension(path: &Path) -> bool { - path.extension() - .is_some_and(|ext| ext.to_string_lossy().eq_ignore_ascii_case("csv")) - } - - /// Parse a file specification into a table name and path - /// - /// # Arguments - /// * `file_spec` - File specification in format [table_name=]file_path - /// - /// # Returns - /// * `SqawkResult<(String, PathBuf)>` - Tuple of (table_name, file_path) - pub fn parse_file_spec(&self, file_spec: &str) -> SqawkResult<(String, PathBuf)> { - // Check for explicit table name in format "table_name=file_path" - if let Some(pos) = file_spec.find('=') { - let (table_name, file_path) = file_spec.split_at(pos); - - // Strip the '=' from the file path - let file_path = &file_path[1..]; - - // "-" names standard input, which has nothing on disk to check for. - let path = PathBuf::from(file_path); - if file_path != STDIN_SPEC && !path.exists() { - return Err(SqawkError::FileNotFound(file_path.to_string())); - } - - Ok((table_name.to_string(), path)) - } else if file_spec == STDIN_SPEC { - // A bare "-" reads standard input into a table called "stdin". - Ok((STDIN_TABLE.to_string(), PathBuf::from(STDIN_SPEC))) - } else { - // No explicit table name, use the file name without extension - let path = PathBuf::from(file_spec); - - // Validate that the file exists - if !path.exists() { - return Err(SqawkError::FileNotFound(file_spec.to_string())); - } - - // Get file name without extension as table name - let file_name = path - .file_name() - .ok_or_else(|| SqawkError::InvalidFileSpec(file_spec.to_string()))? - .to_string_lossy(); - - // Extract name without extension - let table_name = if let Some(pos) = file_name.rfind('.') { - file_name[..pos].to_string() - } else { - file_name.to_string() - }; - - Ok((table_name, path)) - } - } - - /// Get a reference to a table by name - /// - /// # Arguments - /// * `table_name` - Name of the table to retrieve - /// - /// # Returns - /// * `SqawkResult<&Table>` - Reference to the requested table - pub fn get_table(&self, table_name: &str) -> SqawkResult<&Table> { - // SAFETY: The caller of `new` ensures the database outlives this FileHandler - let db = unsafe { &*self.database }; - db.get_table(table_name) - } - - /// Get all table names - /// - /// # Returns - /// * `Vec` - Vector of table names - pub fn table_names(&self) -> Vec { - // SAFETY: The caller of `new` ensures the database outlives this FileHandler - let db = unsafe { &*self.database }; - db.table_names() - } - - /// Get the number of tables - /// - /// # Returns - /// * `usize` - Number of tables - pub fn table_count(&self) -> usize { - // SAFETY: The caller of `new` ensures the database outlives this FileHandler - let db = unsafe { &*self.database }; - db.table_count() - } - /// Explain why `table_name` has no writeback target. fn unwritable_table_error(&self, table_name: &str) -> SqawkError { if self.stdin_table.as_deref() == Some(table_name) { @@ -374,74 +235,62 @@ impl FileHandler { SqawkError::NoFilePath(table_name.to_string()) } - /// Report whether `table_name` could be written back, without writing it. + /// Report whether `table_name` can be written back, without writing it. /// /// Lets a caller reject the whole writeback up front instead of failing /// part-way through and leaving some source files rewritten and others not. - pub fn check_table_writable(&self, table_name: &str) -> SqawkResult<()> { - let table = self.get_table(table_name)?; - if table.file_path().is_none() { - return Err(self.unwritable_table_error(table_name)); - } - Ok(()) - } - - /// Check if a table exists - /// - /// # Arguments - /// * `table_name` - Name of the table to check /// /// # Returns - /// * `bool` - True if the table exists - pub fn has_table(&self, table_name: &str) -> bool { - // SAFETY: The caller of `new` ensures the database outlives this FileHandler - let db = unsafe { &*self.database }; - db.has_table(table_name) + /// * `Ok(true)` - the table has a file and that file can be written + /// * `Ok(false)` - a scratch table (CREATE TABLE without LOCATION, or + /// CREATE TABLE ... AS): it has no file, and writeback skips it + /// * `Err` - a table read from standard input, or a file that is + /// read-only or in a directory that cannot be written + pub fn check_table_writable(&self, db: &Database, table_name: &str) -> SqawkResult { + let table = db.get_table(table_name)?; + match table.file_path() { + Some(path) => { + csv_handler::check_writable(path)?; + Ok(true) + } + None if self.stdin_table.as_deref() == Some(table_name) => { + Err(self.unwritable_table_error(table_name)) + } + None => Ok(false), + } } - /// Save a modified table back to its original file + /// Save a modified table of `db` back to its original file /// /// # Arguments /// * `table_name` - Name of the table to save /// /// # Returns /// * `SqawkResult<()>` - Result of the operation - pub fn save_table(&self, table_name: &str) -> SqawkResult<()> { - // Get a reference to the table - let table = self.get_table(table_name)?; + pub fn save_table(&self, db: &Database, table_name: &str) -> SqawkResult<()> { + let table = db.get_table(table_name)?; if self.config.verbose() { - eprintln!("In FileHandler::save_table for table '{}'", table_name); - - // Access database directly to check if the table exists there - let db = unsafe { &*self.database }; - if let Ok(db_table) = db.get_table(table_name) { - if let Some(path) = db_table.file_path() { - eprintln!( - "Database has table '{}' with file_path '{:?}'", - table_name, path - ); - } else { - eprintln!("Database has table '{}' but NO file_path", table_name); - } - } else { - eprintln!("Table '{}' not found in database", table_name); - } + diag!("In FileHandler::save_table for table '{}'", table_name); } // Check if the table has an associated file path let file_path = match table.file_path() { Some(path) => { if self.config.verbose() { - eprintln!("Table '{}' has file_path '{:?}'", table_name, path); + diag!( + "Table '{}' has file_path '{}'", + table_name, + crate::error::display_path(path) + ); } path } None => { // Log debugging information if self.config.verbose() { - eprintln!("Table '{}' has NO file_path", table_name); - eprintln!( + diag!("Table '{}' has NO file_path", table_name); + diag!( " Table details - Name: {}, Columns: {}, Delimiter: '{}'", table.name(), table.columns().join(","), @@ -457,34 +306,192 @@ impl FileHandler { // Make sure parent directories exist if let Some(parent) = file_path.parent() { if !parent.as_os_str().is_empty() && !parent.exists() { - std::fs::create_dir_all(parent).map_err(SqawkError::IoError)?; + std::fs::create_dir_all(parent).map_err(|e| SqawkError::write(file_path, e))?; } } + // The rename in save_table would replace even a read-only file. + csv_handler::check_writable(file_path)?; csv_handler::save_table(table, file_path) } +} - /// Detect file format based on file extension - /// - /// This method examines the file extension to determine the appropriate handler: - /// - `.csv` files are treated as CSV (comma-separated values) - /// - All other extensions are treated as delimiter-separated files - /// - Files without extensions default to CSV format - /// - /// # Arguments - /// * `path` - File path to analyze - /// - /// # Returns - /// * `FileFormat` - Detected format (Csv or Delimited) based on file extension - fn detect_format(&self, path: &Path) -> FileFormat { - if let Some(ext) = path.extension() { - match ext.to_string_lossy().to_lowercase().as_str() { - "csv" => FileFormat::Csv, - _ => FileFormat::Delimited, - } +/// Fail if `real` (a canonical path) is already some table's file: writeback +/// would make one table's copy overwrite the other's. `shown` names the file +/// in the error. +fn check_not_loaded(db: &Database, real: &Path, shown: &str) -> SqawkResult<()> { + for name in db.table_names() { + let table = db.get_table(&name)?; + if table.file_path().map(PathBuf::as_path) == Some(real) && !table.is_placeholder() { + return Err(SqawkError::InvalidFileSpec(format!( + "file {} is already loaded as table {}", + shown, name + ))); + } + } + Ok(()) +} + +/// How `CREATE TABLE ... LOCATION` reads its file (the `WITH` options). +#[derive(Debug, Clone)] +pub struct LocationOptions { + /// `WITH (DELIMITER=...)`; else tab for `.tsv` / `.tab`, comma otherwise + pub delimiter: Option, + /// The file's first line is a header (`WITH (HEADER=false)` clears it) + pub header: bool, + /// `#` lines are comments (`WITH (COMMENTS=true)`) + pub comments: bool, +} + +/// The table `CREATE TABLE name (schema) LOCATION 'path'` creates. +/// +/// The declared columns are the schema of the file, as for a Hive external +/// table. When the file exists it is loaded through the normal loader under +/// the declared names and types: its first line is a header unless +/// `options.header` is false, and a header must have one field per declared +/// column; it is skipped as data and written back as read. When it does not +/// exist the table starts empty, and `--write` creates the file (with a +/// header line of the declared names unless `options.header` is false). +/// +/// A relative path resolves against the current directory; the table +/// records the absolute path (canonical when the file exists), so a later +/// `.cd` in the REPL does not change which file is written. A file already +/// loaded as another table is an error, as is a directory. +/// +/// # Returns +/// * The table, and whether its file already existed (loading it is not a +/// modification; a new file is created on `--write`) +pub fn load_location( + db: &Database, + table_name: &str, + path: &str, + schema: Vec, + options: &LocationOptions, +) -> SqawkResult<(Table, bool)> { + let given = Path::new(path); + let delimiter = options.delimiter.clone().unwrap_or_else(|| { + if has_tab_extension(given) { + "\t".to_string() } else { - // Default to CSV if no extension - FileFormat::Csv + ",".to_string() + } + }); + let existing = match std::fs::metadata(given) { + Ok(meta) if meta.is_dir() => { + return Err(SqawkError::open( + given, + std::io::Error::new(std::io::ErrorKind::IsADirectory, "Is a directory"), + )) + } + Ok(_) => true, + Err(e) if e.kind() == std::io::ErrorKind::NotFound => false, + Err(e) => return Err(SqawkError::open(given, e)), + }; + let real = if existing { + std::fs::canonicalize(given).map_err(|e| SqawkError::open(given, e))? + } else { + absolute_path(given).map_err(|e| SqawkError::open(given, e))? + }; + check_not_loaded(db, &real, path)?; + + if !existing { + let mut table = Table::new_with_schema(table_name, schema, Some(real), Some(delimiter)); + table.set_file_layout(FileLayout { + has_header: options.header, + ..FileLayout::default() + }); + return Ok((table, false)); + } + + let opts = ReadOptions { + delimiter: delimiter_byte(&delimiter), + header: options.header, + comments: options.comments, + columns: Some(schema.iter().map(|c| c.name.clone()).collect()), + }; + let data = std::fs::read(&real).map_err(|e| SqawkError::open(given, e))?; + let mut table = csv_handler::read_table(table_name, data, Some(real), delimiter, &opts, path)?; + table.set_declared_columns(schema); + Ok((table, true)) +} + +/// `path` made absolute against the current directory, without resolving +/// symlinks in the file name (the file need not exist). An existing parent +/// directory is canonicalized, so two spellings of one new file compare +/// equal. +fn absolute_path(path: &Path) -> std::io::Result { + let abs = std::path::absolute(path)?; + if let (Some(parent), Some(name)) = (abs.parent(), abs.file_name()) { + if let Ok(real_parent) = std::fs::canonicalize(parent) { + return Ok(real_parent.join(name)); + } + } + Ok(abs) +} + +/// Whether `path` names a tab-separated file: extension `.tsv` or `.tab` +/// (any case). +pub(crate) fn has_tab_extension(path: &Path) -> bool { + path.extension().is_some_and(|ext| { + let ext = ext.to_string_lossy(); + ext.eq_ignore_ascii_case("tsv") || ext.eq_ignore_ascii_case("tab") + }) +} + +/// Whether `name` can be the table-name prefix of `NAME=PATH`: non-empty, +/// letters, digits and `_` only (so never a path). +fn is_table_name(name: &str) -> bool { + !name.is_empty() && name.chars().all(|c| c.is_alphanumeric() || c == '_') +} + +/// Split a file operand `[table_name=]file_path` into a table name and path. +/// +/// `NAME=PATH` names the table only when NAME is identifier-like (letters, +/// digits, `_`); otherwise the whole operand is the path, so a file in a +/// directory such as `k=v/` loads as expected. When NAME is identifier-like +/// but PATH does not exist and the whole operand does, the operand is a +/// path too. Without a name, the table is named after the file without its +/// extension; `-` is standard input, as the table `stdin`. +/// +/// Errors: an empty name (`=file.csv`), or a file name that leaves an empty +/// table name (`.csv`). +pub fn parse_table_spec(file_spec: &str) -> SqawkResult<(String, PathBuf)> { + if let Some((name, path)) = file_spec.split_once('=') { + if name.is_empty() { + return Err(SqawkError::InvalidFileSpec(format!( + "'{}': empty table name before '='", + file_spec + ))); } + if is_table_name(name) + && (path == STDIN_SPEC || Path::new(path).exists() || !Path::new(file_spec).exists()) + { + if path.is_empty() { + return Err(SqawkError::InvalidFileSpec(format!( + "'{}': empty file path after '='", + file_spec + ))); + } + return Ok((name.to_string(), PathBuf::from(path))); + } + } + if file_spec == STDIN_SPEC { + return Ok((STDIN_TABLE.to_string(), PathBuf::from(STDIN_SPEC))); + } + let path = PathBuf::from(file_spec); + let file_name = path + .file_name() + .map(|n| n.to_string_lossy().into_owned()) + .unwrap_or_default(); + let table_name = match file_name.rfind('.') { + Some(pos) => file_name[..pos].to_string(), + None => file_name, + }; + if table_name.is_empty() { + return Err(SqawkError::InvalidFileSpec(format!( + "'{}': cannot derive a table name from the file name; use NAME={}", + file_spec, file_spec + ))); } + Ok((table_name, path)) } diff --git a/src/main.rs b/src/main.rs index 948c299..afb6092 100644 --- a/src/main.rs +++ b/src/main.rs @@ -25,6 +25,14 @@ //! 3. Execute SQL statements in sequence //! 4. Print query results to stdout //! 5. Save modified tables back to disk if requested +//! +//! Any error ends the run with one line on stderr, `sqawk: `, and +//! exit status 1 (clap's own usage errors exit with status 2). + +#![forbid(unsafe_code)] + +#[macro_use] +mod diag; mod aggregate; mod capacity; @@ -39,99 +47,180 @@ mod sql_executor; mod table; mod vm; -use anyhow::{Context, Result}; -// Not explicitly importing SqawkArgs as it's not directly used +use std::process::ExitCode; + +use cli::{SqawkArgs, SqlSource}; use config::AppConfig; use database::Database; +use error::{SqawkError, SqawkResult}; use file_handler::FileHandler; use repl::Repl; use sql_executor::SqlExecutor; +/// Stack size sqawk runs on. +/// +/// Parsing, compiling and evaluating an expression recurse once per level of +/// nesting, and SQLite's depth limit (1000, enforced right after parsing) +/// needs more than the 8 MiB main-thread stack in a debug build. The memory +/// is reserved, not committed: pages are touched only as deep as recursion +/// actually goes. +const MAIN_STACK_BYTES: usize = 256 * 1024 * 1024; + +/// Parse the command line, then run [`run`] on a freshly mapped +/// [`MAIN_STACK_BYTES`] stack, on this thread. (A second thread with a large +/// stack would work too, but once a process has two threads glibc's malloc +/// takes a lock on every call, which slows a large load or GROUP BY by +/// several percent.) +fn main() -> ExitCode { + let args = cli::parse_args(); + match stacker::grow(MAIN_STACK_BYTES, || run(args)) { + Ok(status) => ExitCode::from(status), + Err(e) => { + diag!("sqawk: {}", one_line(&e.to_string())); + ExitCode::FAILURE + } + } +} + +/// `msg` on a single line: line breaks (which an error quoting multi-line +/// SQL could contain) become spaces, so every error is exactly one line. +pub fn one_line(msg: &str) -> String { + if !msg.contains(['\n', '\r']) { + return msg.to_string(); + } + msg.replace("\r\n", " ").replace(['\n', '\r'], " ") +} + +/// Whether a file operand reads standard input. +fn reads_stdin(spec: &str) -> bool { + file_handler::parse_table_spec(spec) + .is_ok_and(|(_, path)| path.as_os_str() == file_handler::STDIN_SPEC) +} + +/// The text of every `-s` / `-f` argument, in command-line order. Files are +/// read up front, so a mistyped `-f` path fails before anything runs. +fn read_script(args: &SqawkArgs) -> SqawkResult> { + let sql_from_stdin = args + .script + .iter() + .any(|s| matches!(s, SqlSource::File(f) if f == file_handler::STDIN_SPEC)); + if sql_from_stdin && args.files.iter().any(|spec| reads_stdin(spec)) { + return Err(SqawkError::InvalidFileSpec( + "standard input cannot supply both SQL (-f -) and a table (-)".to_string(), + )); + } + let mut stdin_read = false; + let texts: Vec<(bool, String)> = args + .script + .iter() + .map(|source| { + let is_file = matches!(source, SqlSource::File(_)); + read_source(source, &mut stdin_read).map(|text| (is_file, text)) + }) + .collect::>()?; + // A file with no statements (empty, or only whitespace) runs nothing; + // an empty `-s` is still an error. + Ok(texts + .into_iter() + .filter(|(is_file, text)| !(*is_file && text.trim().is_empty())) + .map(|(_, text)| text) + .collect()) +} + +/// The SQL text of one `-s` / `-f` argument. +fn read_source(source: &SqlSource, stdin_read: &mut bool) -> SqawkResult { + match source { + SqlSource::Text(sql) => Ok(sql.clone()), + SqlSource::File(path) if path == file_handler::STDIN_SPEC => { + if *stdin_read { + return Err(SqawkError::InvalidFileSpec( + "standard input can only be read once; '-f -' was given more than once" + .to_string(), + )); + } + *stdin_read = true; + let mut sql = String::new(); + std::io::Read::read_to_string(&mut std::io::stdin().lock(), &mut sql) + .map_err(|e| SqawkError::open("standard input", e))?; + Ok(sql) + } + SqlSource::File(path) => { + std::fs::read_to_string(path).map_err(|e| SqawkError::open(path, e)) + } + } +} + /// Main entry point for the sqawk utility /// /// This function orchestrates the entire application flow: -/// 1. Parses command line arguments -/// 2. Loads input files according to specifications -/// 3. Executes SQL statements against the in-memory tables -/// 4. Outputs results to stdout -/// 5. Handles errors with context for better diagnostics -/// 6. Optionally saves modified tables back to disk +/// 1. Loads input files according to specifications +/// 2. Executes SQL statements against the in-memory tables +/// 3. Outputs results to stdout +/// 4. Optionally saves modified tables back to disk /// /// The design follows a functional pipeline approach: /// - Files → In-memory Tables → SQL Processing → Results → (Optional) File Writeback /// /// # Returns -/// * `Ok(())` if all operations completed successfully -/// * `Err` with context if any step fails -fn main() -> Result<()> { - // Step 1: Parse command-line arguments - // This handles -s/--sql, file specs, -F (field separator), --write, and -v flags - let args = cli::parse_args()?; - - // Step 1b: Create a centralized application configuration +/// * `Ok(status)` with the exit status: 0 for a completed run, or what the +/// REPL session ended with (see [`Repl::run`]) +/// * `Err` with the first error; `main` prints it as one line +fn run(args: SqawkArgs) -> SqawkResult { + // Create a centralized application configuration // This will be passed to all components that need configuration settings - let field_separator = args - .field_separator - .as_deref() - .map(cli::validate_separator) - .transpose()?; let config = AppConfig::new( - args.verbose, // Verbose output flag - field_separator, // Field separator for tables - args.tabledef.clone(), // Table column definitions - args.write, // Whether to write changes to files + args.verbose, // Verbose output flag + args.field_separator.clone(), // Field separator for tables + args.tabledef.clone(), // Table column definitions + args.write, // Whether to write changes to files ) - .with_no_header(args.no_header); + .with_no_header(args.no_header) + .with_comments(args.comments) + .with_output_delimiter(args.output_delimiter.clone()) + .with_no_output_header(args.no_output_header); - // Configure diagnostics output if verbose mode is enabled (-v flag) - // This is important for debugging and understanding the execution flow + // Debugging output (-v); its format is not stable. if config.verbose() { - eprintln!("Running in verbose mode"); - eprintln!("Arguments: {args:?}"); + diag!("Running in verbose mode"); + diag!("Arguments: {args:?}"); } - // Step 2a: Create a new Database instance to serve as the central store for tables + let script = read_script(&args)?; + + // Create a new Database instance to serve as the central store for tables let mut database = Database::new(); - // Step 2b: Compile table definitions from the CLI arguments + // Compile table definitions from the CLI arguments // This makes Database the source of truth for table schemas defined via CLI - database - .compile_table_definitions(&config) - .context("Failed to compile table definitions")?; + database.compile_table_definitions(&config)?; - // Step 2c: Initialize the file handler with the application configuration + // Initialize the file handler with the application configuration // The file handler uses config for field separator and verbosity - // Note: table definitions are now handled by Database directly - let mut file_handler = FileHandler::new(&config, &mut database); + let mut file_handler = FileHandler::new(&config); // The REPL reads its commands from stdin, so it cannot also consume stdin // as table data: whichever got there first would starve the other. - if args.interactive - && args.files.iter().any(|spec| { - // Same split as parse_file_spec: table name is before the first '='. - spec.split_once('=').map_or(spec.as_str(), |(_, path)| path) == file_handler::STDIN_SPEC - }) - { - anyhow::bail!( + if args.interactive && args.files.iter().any(|spec| reads_stdin(spec)) { + return Err(SqawkError::InvalidFileSpec( "cannot read a table from standard input in interactive mode: \ the REPL reads its own commands from stdin" - ); + .to_string(), + )); } - // Step 2b: Load all specified files into in-memory tables + // Load all specified files into in-memory tables // Each file can specify its table name with table_name=file_path syntax for file_spec in &args.files { - file_handler - .load_file(file_spec) - .with_context(|| format!("Failed to load file: {file_spec}"))?; + file_handler.load_file(&mut database, file_spec)?; } + database.check_table_definitions_used()?; // Log table loading results in verbose mode if config.verbose() { - let table_count = file_handler.table_count(); - eprintln!("Loaded {table_count} tables"); - for table_name in file_handler.table_names() { - eprintln!("Table '{table_name}' loaded"); + let table_count = database.table_count(); + diag!("Loaded {table_count} tables"); + for table_name in database.table_names() { + diag!("Table '{table_name}' loaded"); } } @@ -141,32 +230,28 @@ fn main() -> Result<()> { file_handler.disable_stdin(); } - // Step 3: Create SQL executor + // Create SQL executor, which takes ownership of the database and file handler // The executor maintains state across statements, allowing multi-statement operations - let mut sql_executor = SqlExecutor::new(&mut database, &mut file_handler, &config); + let mut sql_executor = SqlExecutor::new(database, file_handler, &config); // Check if interactive mode is requested if args.interactive { // Start REPL (Read-Eval-Print Loop) for interactive SQL entry let mut repl = Repl::new(sql_executor, &config); - match repl.run() { - Ok(_) => return Ok(()), - Err(e) => return Err(anyhow::anyhow!("Failed to run interactive mode: {}", e)), - } + return Ok(repl.run()); } let mut stdout_open = true; + let header = !config.no_output_header(); // Process each SQL statement in the order specified on the command line // This allows operations like: UPDATE -> DELETE -> SELECT to see the effects - for sql in &args.sql { + for sql in &script { // Execute the SQL statement against the in-memory tables // The result may be a table (for SELECT) or None (for UPDATE, DELETE, INSERT) - let result = sql_executor - .execute(sql) - .with_context(|| format!("Failed to execute SQL: {sql}"))?; + let result = sql_executor.execute(sql)?; - // Step 4: Output results to stdout (for SELECT queries). + // Output results to stdout (for SELECT queries). // // One result set per statement, each with its own header. They used to // share a single buffer and schema, so `SELECT a; SELECT b` printed @@ -174,43 +259,42 @@ fn main() -> Result<()> { for table in &result { if config.verbose() { let row_count = table.row_count(); - eprintln!("Query returned {row_count} rows"); + diag!("Query returned {row_count} rows"); } // Print the result table in delimiter-separated format. Once the // reader of a pipe has gone away (`sqawk ... | head`), stop // printing but keep going: later statements and --write still run. if stdout_open { - if let Err(e) = table.print_to_stdout() { + if let Err(e) = table.print_to_stdout(header) { if e.kind() != std::io::ErrorKind::BrokenPipe { - return Err(e).context("Failed to write results"); + return Err(SqawkError::write("standard output", e)); } stdout_open = false; } } } if result.is_empty() && config.verbose() { - eprintln!("Query executed successfully (no results to display)"); + diag!("Query executed successfully (no results to display)"); } } - // Step 5: Handle file writeback based on the configuration + // A transaction left open by the last statement is rolled back, so its + // changes are neither written back nor kept. + sql_executor.finish(); + + // Handle file writeback based on the configuration // By default, Sqawk operates in read-only mode unless explicitly told to write if config.write_changes() { - // Only tables that were actually modified (by UPDATE, INSERT, DELETE) - // will be written back to their source files - sql_executor - .save_modified_tables() - .context("Failed to save modified tables")?; + // Only modified tables with a file are written back + sql_executor.save_modified_tables()?; } else if config.verbose() { // In verbose mode, remind the user that changes weren't saved - eprintln!("Changes not saved: use --write to save changes to files"); + diag!("Changes not saved: use --write to save changes to files"); } // The process is about to exit, so freeing every loaded cell one by one // is wasted work (measurable on large inputs); let the OS reclaim it. - drop(sql_executor); - drop(file_handler); - std::mem::forget(database); + std::mem::forget(sql_executor); - Ok(()) + Ok(0) } diff --git a/src/repl.rs b/src/repl.rs index 051ebb2..87b9d38 100644 --- a/src/repl.rs +++ b/src/repl.rs @@ -1,3 +1,17 @@ +//! The interactive shell (`sqawk -i`). +//! +//! When standard input is a terminal the REPL is a line editor: banner, +//! `sqawk> ` prompt, history, tab completion, and status messages after +//! commands. When standard input is not a terminal (a piped or redirected +//! script) it reads the input line by line and standard output carries only +//! what the commands produce: query results and the output of `.tables`, +//! `.schema`, `.print` and the other reporting commands. Status messages are +//! then dropped (written to standard error under `-v`). +//! +//! Errors are printed as `sqawk: ` on standard error and the +//! session continues. A script that hit any error exits with status 1; +//! `.exit N` exits with status N in both modes. + use rustyline::completion::{Completer, Pair}; use rustyline::config::CompletionType; use rustyline::error::ReadlineError; @@ -7,46 +21,38 @@ use rustyline::history::DefaultHistory; use rustyline::validate::{self, Validator}; use rustyline::{Config, Context, Editor, Helper}; use std::borrow::Cow; +use std::ffi::OsString; use std::fmt; +use std::io::{BufRead, IsTerminal}; +use std::path::PathBuf; use crate::config::AppConfig; use crate::error::SqawkError; use crate::sql_executor::SqlExecutor; +use crate::vm::like::{Case, Pattern}; -// Define a custom error type for the REPL +/// An error from one REPL command. Its `Display` is the bare message; the +/// REPL prints it as `sqawk: `, the CLI's error form. #[derive(Debug)] pub enum ReplError { - SqlExecutor(anyhow::Error), - Readline(ReadlineError), Io(std::io::Error), Sqawk(SqawkError), + /// A malformed or failed dot-command. + Command(String), } impl fmt::Display for ReplError { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { match self { - ReplError::SqlExecutor(err) => write!(f, "SQL execution error: {}", err), - ReplError::Readline(err) => write!(f, "Input error: {}", err), - ReplError::Io(err) => write!(f, "I/O error: {}", err), - ReplError::Sqawk(err) => write!(f, "Sqawk error: {}", err), + ReplError::Io(err) => write!(f, "{}", crate::error::io_reason(err)), + ReplError::Sqawk(err) => write!(f, "{}", err), + ReplError::Command(msg) => write!(f, "{}", msg), } } } impl std::error::Error for ReplError {} -impl From for ReplError { - fn from(err: anyhow::Error) -> Self { - ReplError::SqlExecutor(err) - } -} - -impl From for ReplError { - fn from(err: ReadlineError) -> Self { - ReplError::Readline(err) - } -} - impl From for ReplError { fn from(err: std::io::Error) -> Self { ReplError::Io(err) @@ -61,32 +67,326 @@ impl From for ReplError { pub type Result = std::result::Result; -const HISTORY_FILE: &str = ".sqawk_history"; - -/// Command completer for REPL commands -#[derive(Default)] -struct CommandCompleter { - /// List of available dot commands for auto-completion - commands: Vec, +/// Every dot-command, for completion. +const DOT_COMMANDS: &[&str] = &[ + ".cd", ".changes", ".exit", ".help", ".load", ".print", ".quit", ".save", ".schema", ".show", + ".stats", ".tables", ".version", ".write", +]; + +/// Dot-commands whose argument is a table name. +const TABLE_ARG_COMMANDS: &[&str] = &[".save", ".schema", ".tables"]; + +/// SQL keywords offered by tab completion: the words of the statements and +/// clauses sqawk implements. +const SQL_KEYWORDS: &[&str] = &[ + "ADD", + "ALL", + "ALTER", + "AND", + "AS", + "ASC", + "BEGIN", + "BETWEEN", + "BY", + "CASE", + "CAST", + "COLUMN", + "COMMIT", + "CREATE", + "CROSS", + "CURRENT", + "DELETE", + "DESC", + "DISTINCT", + "DROP", + "ELSE", + "END", + "ESCAPE", + "EXCEPT", + "EXISTS", + "FALSE", + "FILTER", + "FIRST", + "FOLLOWING", + "FROM", + "FULL", + "GROUP", + "HAVING", + "IF", + "ILIKE", + "IN", + "INNER", + "INSERT", + "INTEGER", + "INTERSECT", + "INTO", + "IS", + "JOIN", + "LAST", + "LEFT", + "LIKE", + "LIMIT", + "LOCATION", + "NATURAL", + "NOT", + "NULL", + "NULLS", + "OFFSET", + "ON", + "OR", + "ORDER", + "OUTER", + "OVER", + "PARTITION", + "PRECEDING", + "RANGE", + "REAL", + "RENAME", + "RIGHT", + "ROLLBACK", + "ROW", + "ROWS", + "SELECT", + "SET", + "TABLE", + "TEXT", + "THEN", + "TO", + "TRANSACTION", + "TRUE", + "TRUNCATE", + "UNBOUNDED", + "UNION", + "UPDATE", + "USING", + "VALUES", + "WHEN", + "WHERE", + // Aggregate and window functions (scalar functions come from the + // function table). + "AVG", + "COUNT", + "CUME_DIST", + "DENSE_RANK", + "FIRST_VALUE", + "GROUP_CONCAT", + "LAG", + "LAST_VALUE", + "LEAD", + "MAX", + "MIN", + "NTH_VALUE", + "NTILE", + "PERCENT_RANK", + "RANK", + "ROW_NUMBER", + "STRING_AGG", + "SUM", + "TOTAL", +]; + +/// The loaded tables and their columns, as completion sees them. +type Schema = Vec<(String, Vec)>; + +/// Whether `name` starts with `prefix`, ignoring ASCII case. +fn starts_with_ci(name: &str, prefix: &str) -> bool { + name.len() >= prefix.len() + && name.is_char_boundary(prefix.len()) + && name[..prefix.len()].eq_ignore_ascii_case(prefix) } -impl CommandCompleter { - /// Create a new command completer with the list of available commands - fn new() -> Self { - let commands = vec![ - ".cd", ".changes", ".exit", ".help", ".load", ".print", ".quit", ".save", ".schema", - ".show", ".stats", ".tables", ".version", ".write", - ] - .into_iter() - .map(|s| s.to_string()) +/// The completions for the cursor at byte offset `pos` of `line`: the byte +/// offset the replacement starts at, and the candidates. +/// +/// * A dot-command name completes from [`DOT_COMMANDS`]; the argument of +/// `.schema`, `.save` and `.tables` completes table names. +/// * `table.prefix` completes the columns of `table`. +/// * Any other word completes table names, column names, SQL keywords and +/// function names. Keywords follow the case of the typed word (lower case +/// if it is all lower case, else upper case). +/// * An empty word completes table names. +/// +/// Matching is a case-insensitive prefix match. +fn complete_line(line: &str, pos: usize, schema: &Schema) -> (usize, Vec) { + let pos = pos.min(line.len()); + let before = &line[..pos]; + let trimmed = before.trim_start(); + + if trimmed.starts_with('.') { + let lead = before.len() - trimmed.len(); + return match trimmed.find(char::is_whitespace) { + None => { + let cands = DOT_COMMANDS + .iter() + .filter(|c| starts_with_ci(c, trimmed)) + .map(|c| c.to_string()) + .collect(); + (lead, cands) + } + Some(end) => { + let cmd = trimmed[..end].to_ascii_lowercase(); + if !TABLE_ARG_COMMANDS.contains(&cmd.as_str()) { + return (pos, Vec::new()); + } + let word_start = before.rfind(char::is_whitespace).map_or(0, |i| { + i + before[i..].chars().next().map_or(1, char::len_utf8) + }); + let prefix = &before[word_start..]; + let cands = schema + .iter() + .filter(|(t, _)| starts_with_ci(t, prefix)) + .map(|(t, _)| t.clone()) + .collect(); + (word_start, cands) + } + }; + } + + // The word under the cursor: identifier characters and dots. + let start = before + .char_indices() + .rev() + .take_while(|(_, c)| c.is_alphanumeric() || *c == '_' || *c == '.') + .last() + .map_or(pos, |(i, _)| i); + let word = &before[start..]; + + if let Some(dot) = word.rfind('.') { + let qualifier = &word[..dot]; + let prefix = &word[dot + 1..]; + let cands = schema + .iter() + .find(|(t, _)| t.eq_ignore_ascii_case(qualifier)) + .map(|(_, cols)| { + cols.iter() + .filter(|c| starts_with_ci(c, prefix)) + .cloned() + .collect() + }) + .unwrap_or_default(); + return (start + dot + 1, cands); + } + + let mut cands: Vec = Vec::new(); + let mut push = |s: String| { + if !cands.iter().any(|c| c.eq_ignore_ascii_case(&s)) { + cands.push(s); + } + }; + for (table, _) in schema { + if starts_with_ci(table, word) { + push(table.clone()); + } + } + if word.is_empty() { + return (start, cands); + } + for (_, cols) in schema { + for col in cols { + if starts_with_ci(col, word) { + push(col.clone()); + } + } + } + let lower = word.chars().any(|c| c.is_ascii_lowercase()) + && !word.chars().any(|c| c.is_ascii_uppercase()); + let mut words: Vec<&str> = SQL_KEYWORDS + .iter() + .copied() + .chain(crate::vm::scalar_function_names()) + .filter(|k| starts_with_ci(k, word)) .collect(); + words.sort_unstable(); + for k in words { + push(if lower { + k.to_ascii_lowercase() + } else { + k.to_string() + }); + } + (start, cands) +} + +/// What a piece of SQL text holds, ignoring whitespace and comments. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +struct SqlScan { + /// It holds something other than whitespace, comments and `;`. + has_tokens: bool, + /// Its last token is a `;` outside any quote or comment. + ends_with_semicolon: bool, +} - Self { commands } +/// Scan `sql` for [`SqlScan`]: `'...'`, `"..."`, `` `...` `` and `[...]` +/// quote, `--` and `/* */` are comments. +fn scan_sql(sql: &str) -> SqlScan { + let mut has_tokens = false; + let mut semi = false; + let mut chars = sql.chars().peekable(); + while let Some(c) = chars.next() { + match c { + c if c.is_whitespace() => {} + '-' if chars.peek() == Some(&'-') => { + for c in chars.by_ref() { + if c == '\n' { + break; + } + } + } + '/' if chars.peek() == Some(&'*') => { + chars.next(); + let mut prev = '\0'; + for c in chars.by_ref() { + if prev == '*' && c == '/' { + break; + } + prev = c; + } + } + ';' => semi = true, + '\'' | '"' | '`' | '[' => { + has_tokens = true; + semi = false; + let close = if c == '[' { ']' } else { c }; + for c in chars.by_ref() { + if c == close { + break; + } + } + } + _ => { + has_tokens = true; + semi = false; + } + } + } + SqlScan { + has_tokens, + ends_with_semicolon: semi, } } -// Implement the required traits for Helper -impl Completer for CommandCompleter { +/// The history file for `XDG_STATE_HOME` / `HOME`: `$XDG_STATE_HOME/sqawk/history` +/// when `XDG_STATE_HOME` is an absolute path, else `$HOME/.sqawk_history` +/// when `HOME` is set and non-empty, else none. +fn history_path(xdg_state_home: Option, home: Option) -> Option { + if let Some(state) = xdg_state_home + .map(PathBuf::from) + .filter(|p| p.is_absolute()) + { + return Some(state.join("sqawk").join("history")); + } + home.filter(|h| !h.is_empty()) + .map(|h| PathBuf::from(h).join(".sqawk_history")) +} + +/// The rustyline helper: completion and multi-line statement entry. +#[derive(Default)] +struct ReplHelper { + /// The loaded tables and columns, refreshed before every prompt. + schema: Schema, +} + +impl Completer for ReplHelper { type Candidate = Pair; fn complete( @@ -95,34 +395,21 @@ impl Completer for CommandCompleter { pos: usize, _ctx: &Context<'_>, ) -> rustyline::Result<(usize, Vec)> { - // Only provide completion for dot commands - if line.starts_with('.') { - // Split the input into command and argument parts - let parts: Vec<&str> = line.splitn(2, ' ').collect(); - let partial_cmd = parts[0]; - - // We're completing a command (not an argument) - let start_pos = 0; // Start of the command - let candidates: Vec = self - .commands - .iter() - .filter(|cmd| cmd.starts_with(partial_cmd)) - .map(|cmd| Pair { - display: cmd.clone(), - replacement: cmd.clone(), + let (start, cands) = complete_line(line, pos, &self.schema); + Ok(( + start, + cands + .into_iter() + .map(|c| Pair { + display: c.clone(), + replacement: c, }) - .collect(); - - Ok((start_pos, candidates)) - } else { - // No completion for SQL statements for now - Ok((pos, vec![])) - } + .collect(), + )) } } -// Implement minimal no-op versions of the other traits required by Helper -impl Hinter for CommandCompleter { +impl Hinter for ReplHelper { type Hint = String; fn hint(&self, _line: &str, _pos: usize, _ctx: &Context<'_>) -> Option { @@ -130,7 +417,7 @@ impl Hinter for CommandCompleter { } } -impl Highlighter for CommandCompleter { +impl Highlighter for ReplHelper { fn highlight<'l>(&self, line: &'l str, _pos: usize) -> Cow<'l, str> { Cow::Borrowed(line) } @@ -140,40 +427,32 @@ impl Highlighter for CommandCompleter { } } -impl Validator for CommandCompleter { +impl Validator for ReplHelper { + /// Input is complete when it is blank, a dot-command, or SQL ending in + /// `;`; otherwise Enter continues the statement on a new line. fn validate( &self, ctx: &mut validate::ValidationContext, ) -> rustyline::Result { - let input = ctx.input().trim(); - - // Empty input is valid (just press enter) - if input.is_empty() { - return Ok(validate::ValidationResult::Valid(None)); - } - - // Dot commands are complete on a single line - if input.starts_with('.') { - return Ok(validate::ValidationResult::Valid(None)); - } - - // SQL statements must end with semicolon - if input.ends_with(';') { - return Ok(validate::ValidationResult::Valid(None)); + let input = ctx.input(); + let scan = scan_sql(input); + if input.trim_start().starts_with('.') || !scan.has_tokens || scan.ends_with_semicolon { + Ok(validate::ValidationResult::Valid(None)) + } else { + Ok(validate::ValidationResult::Incomplete) } - - // Otherwise, we need more input (multiline SQL) - Ok(validate::ValidationResult::Incomplete) } } -impl Helper for CommandCompleter {} +impl Helper for ReplHelper {} /// Commands that can be executed in the REPL #[derive(Debug)] enum ReplCommand { /// Execute SQL statement Sql(String), + /// Blank input or only comments: nothing to do + Nothing, /// Load a file into a table Load(String), /// Show the list of tables matching an optional pattern @@ -200,197 +479,250 @@ enum ReplCommand { Show(Option), /// Show statistics or toggle statistics mode Stats(Option), - /// Unknown command - Unknown(String), + /// A malformed or unknown dot-command (the error message) + Invalid(String), } /// REPL interface for interactive SQL entry -pub struct Repl<'a> { +pub struct Repl { /// SQL executor for running queries - executor: SqlExecutor<'a>, - /// Rustyline editor for command line editing - editor: Editor, + executor: SqlExecutor, /// Application configuration for global settings config: AppConfig, + /// Standard input is a terminal: line editing, prompt, banner, history + /// and status messages + interactive: bool, + /// The line editor (interactive only) + editor: Option>, + /// Where history is kept (interactive only, when a location exists) + history: Option, /// Whether the REPL is running running: bool, + /// The status `.exit N` asked for + exit_code: Option, + /// A command failed (makes a script's exit status 1) + failed: bool, /// Whether to show number of rows changed by SQL statements show_changes: bool, /// Whether to show query statistics show_stats: bool, } -impl<'a> Repl<'a> { +impl Repl { /// Create a new REPL - pub fn new(executor: SqlExecutor<'a>, app_config: &AppConfig) -> Self { - // Create rustyline configuration with list-style completion - let rustyline_config = Config::builder() - .completion_type(CompletionType::List) - .build(); - - // Create editor with our custom command completer - let helper = CommandCompleter::new(); - let mut editor = Editor::with_config(rustyline_config).expect("Failed to create editor"); - - // Set the helper manually - editor.set_helper(Some(helper)); - - // Load history if available - let _ = editor.load_history(HISTORY_FILE); + pub fn new(executor: SqlExecutor, app_config: &AppConfig) -> Self { + let interactive = std::io::stdin().is_terminal(); + let mut editor = None; + let mut history = None; + if interactive { + let rustyline_config = Config::builder() + .completion_type(CompletionType::List) + .auto_add_history(false) + .build(); + match Editor::with_config(rustyline_config) { + Ok(mut ed) => { + ed.set_helper(Some(ReplHelper::default())); + history = + history_path(std::env::var_os("XDG_STATE_HOME"), std::env::var_os("HOME")); + if let Some(path) = &history { + let _ = ed.load_history(path); + } + editor = Some(ed); + } + Err(e) => diag!("sqawk: cannot start the line editor: {}", e), + } + } Self { executor, - editor, config: app_config.clone(), + interactive, + editor, + history, running: true, - show_changes: false, // Default to not showing changes - show_stats: false, // Default to not showing stats + exit_code: None, + failed: false, + show_changes: false, + show_stats: false, } } - /// Run the REPL - pub fn run(&mut self) -> Result<()> { - println!("Welcome to Sqawk interactive mode!"); - println!("Enter SQL statements or commands, terminate with ';'."); - println!("Type .help for available commands."); + /// Run the REPL until `.exit`, `.quit` or end of input, and return the + /// process exit status: the `.exit N` code if one was given, else 1 for + /// a script in which any command failed, else 0. + pub fn run(&mut self) -> u8 { + if self.editor.is_some() { + outln!("Welcome to Sqawk interactive mode!"); + outln!("Enter SQL statements or commands, terminate with ';'."); + outln!("Type .help for available commands."); + self.run_editor(); + } else { + self.run_script(); + } + + // Uncommitted work is discarded, as when SQLite closes a connection. + self.executor.finish(); + if let (Some(editor), Some(path)) = (self.editor.as_mut(), &self.history) { + let saved = match path.parent() { + Some(dir) => std::fs::create_dir_all(dir).map_err(ReadlineError::from), + None => Ok(()), + } + .and_then(|()| editor.save_history(path)); + if let Err(e) = saved { + diag!( + "sqawk: cannot save history to '{}': {}", + path.display(), + crate::one_line(&e.to_string()) + ); + } + } + + match self.exit_code { + Some(code) => code, + None if self.failed && !self.interactive => 1, + None => 0, + } + } + + /// The terminal loop: one line-edited entry (possibly several lines of a + /// statement) per command. + fn run_editor(&mut self) { while self.running { - match self.read_command() { - Ok(command) => { - if let Err(e) = self.execute_command(command) { - eprintln!("Error: {}", e); + let schema = self.schema(); + let Some(editor) = self.editor.as_mut() else { + return; + }; + if let Some(helper) = editor.helper_mut() { + helper.schema = schema; + } + match editor.readline("sqawk> ") { + Ok(input) => { + if !input.trim().is_empty() { + let _ = editor.add_history_entry(input.as_str()); } + self.dispatch(&input); } - Err(ReadlineError::Interrupted) => { - println!("CTRL-C"); - break; - } - Err(ReadlineError::Eof) => { - println!("CTRL-D"); - break; - } + // Ctrl-C abandons the entry being typed. + Err(ReadlineError::Interrupted) => {} + Err(ReadlineError::Eof) => break, Err(err) => { - eprintln!("Error: {:?}", err); + diag!("sqawk: {}", crate::one_line(&err.to_string())); break; } } } + } - self.editor.save_history(HISTORY_FILE).unwrap_or_else(|e| { - eprintln!("Failed to save history: {}", e); - }); - - Ok(()) + /// The script loop: read standard input line by line. A dot-command is a + /// line of its own; SQL runs once the text gathered ends with `;`, and a + /// final statement without `;` runs at end of input. + fn run_script(&mut self) { + let stdin = std::io::stdin(); + let mut reader = stdin.lock(); + let mut pending = String::new(); + let mut raw = Vec::new(); + while self.running { + raw.clear(); + match reader.read_until(b'\n', &mut raw) { + Ok(0) => break, + Ok(_) => {} + Err(e) => { + diag!( + "sqawk: cannot read standard input: {}", + crate::error::io_reason(&e) + ); + self.failed = true; + break; + } + } + let text = String::from_utf8_lossy(&raw); + let line = text.trim_end_matches(['\n', '\r']); + if !scan_sql(&pending).has_tokens && line.trim_start().starts_with('.') { + pending.clear(); + self.dispatch(line); + continue; + } + pending.push_str(line); + pending.push('\n'); + if scan_sql(&pending).ends_with_semicolon { + let sql = std::mem::take(&mut pending); + self.dispatch(&sql); + } + } + if self.running { + self.dispatch(&pending); + } } - /// Read a command from the user - fn read_command(&mut self) -> rustyline::Result { - let prompt = "sqawk> "; - let input = self.editor.readline(prompt)?; + /// The loaded tables and their columns. + fn schema(&self) -> Schema { + self.executor + .table_names() + .into_iter() + .map(|t| { + let cols = self.executor.get_table_columns(&t).unwrap_or_default(); + (t, cols) + }) + .collect() + } - if !input.trim().is_empty() { - let _ = self.editor.add_history_entry(&input); + /// Parse and run one command, reporting its error. + fn dispatch(&mut self, input: &str) { + let command = self.parse_command(input); + if let Err(e) = self.execute_command(command) { + diag!("sqawk: {}", crate::one_line(&e.to_string())); + self.failed = true; } + } - Ok(self.parse_command(&input)) + /// A status message (a confirmation or reminder rather than requested + /// output): on standard output in a terminal session; in a script, on + /// standard error under `-v` and otherwise dropped. + fn note(&self, msg: fmt::Arguments<'_>) { + if self.interactive { + crate::diag::out_line(msg); + } else if self.config.verbose() { + crate::diag::write_line(msg); + } } /// Parse a command from user input fn parse_command(&self, input: &str) -> ReplCommand { - let input = input.trim(); + let trimmed = input.trim(); - if let Some(stripped) = input.strip_prefix('.') { + if let Some(stripped) = trimmed.strip_prefix('.') { let parts: Vec<&str> = stripped.splitn(2, ' ').collect(); let command = parts[0].to_lowercase(); + let arg = parts.get(1).map(|a| a.trim().to_string()); match command.as_str() { - "exit" => { - if parts.len() > 1 { - ReplCommand::Exit(Some(parts[1].trim().to_string())) - } else { - ReplCommand::Exit(None) - } - } + "exit" => ReplCommand::Exit(arg), "quit" => ReplCommand::Exit(None), - "tables" => { - if parts.len() > 1 { - ReplCommand::Tables(Some(parts[1].trim().to_string())) - } else { - ReplCommand::Tables(None) - } - } - "schema" => { - if parts.len() > 1 { - ReplCommand::Schema(Some(parts[1].trim().to_string())) - } else { - // With no argument, show schema for all tables - ReplCommand::Schema(None) - } - } - "load" => { - if parts.len() > 1 { - ReplCommand::Load(parts[1].trim().to_string()) - } else { - ReplCommand::Unknown("File path required for .load command".to_string()) - } - } - "write" => { - if parts.len() > 1 { - ReplCommand::WriteMode(Some(parts[1].trim().to_string())) - } else { - ReplCommand::WriteMode(None) - } - } - "cd" => { - if parts.len() > 1 { - ReplCommand::ChangeDirectory(parts[1].trim().to_string()) - } else { - ReplCommand::Unknown("Directory path required for .cd command".to_string()) - } - } - "changes" => { - if parts.len() > 1 { - ReplCommand::Changes(Some(parts[1].trim().to_string())) - } else { - ReplCommand::Changes(None) - } - } - "print" => { - if parts.len() > 1 { - ReplCommand::Print(parts[1].to_string()) - } else { - ReplCommand::Print("".to_string()) // Print an empty line - } - } + "tables" => ReplCommand::Tables(arg), + "schema" => ReplCommand::Schema(arg), + "load" => match arg { + Some(a) if !a.is_empty() => ReplCommand::Load(a), + _ => ReplCommand::Invalid("usage: .load [TABLE=]FILE".to_string()), + }, + "write" => ReplCommand::WriteMode(arg), + "cd" => match arg { + Some(a) if !a.is_empty() => ReplCommand::ChangeDirectory(a), + _ => ReplCommand::Invalid("usage: .cd DIRECTORY".to_string()), + }, + "changes" => ReplCommand::Changes(arg), + "print" => ReplCommand::Print(parts.get(1).unwrap_or(&"").to_string()), "version" => ReplCommand::Version, "help" => ReplCommand::Help, - "save" => { - if parts.len() > 1 { - ReplCommand::Save(Some(parts[1].trim().to_string())) - } else { - ReplCommand::Save(None) - } - } - "show" => { - if parts.len() > 1 { - ReplCommand::Show(Some(parts[1].trim().to_string())) - } else { - ReplCommand::Show(None) - } - } - "stats" => { - if parts.len() > 1 { - ReplCommand::Stats(Some(parts[1].trim().to_string())) - } else { - ReplCommand::Stats(None) - } - } - _ => ReplCommand::Unknown(format!("Unknown command: .{}", command)), + "save" => ReplCommand::Save(arg), + "show" => ReplCommand::Show(arg), + "stats" => ReplCommand::Stats(arg), + _ => ReplCommand::Invalid(format!("unknown command: .{}", command)), } - } else if !input.is_empty() { + } else if scan_sql(input).has_tokens { ReplCommand::Sql(input.to_string()) } else { - ReplCommand::Unknown("Empty command".to_string()) + ReplCommand::Nothing } } @@ -398,6 +730,7 @@ impl<'a> Repl<'a> { fn execute_command(&mut self, command: ReplCommand) -> Result<()> { match command { ReplCommand::Sql(sql) => self.execute_sql(&sql), + ReplCommand::Nothing => Ok(()), ReplCommand::Load(file_spec) => self.load_file(&file_spec), ReplCommand::Tables(pattern) => self.show_tables(pattern.as_deref()), ReplCommand::Schema(table_name) => self.show_schema(table_name.as_deref()), @@ -410,14 +743,11 @@ impl<'a> Repl<'a> { ReplCommand::Show(option) => self.show_settings(option.as_deref()), ReplCommand::Stats(option) => self.show_stats(option.as_deref()), ReplCommand::Print(text) => { - println!("{}", text); + outln!("{}", text); Ok(()) } ReplCommand::Version => self.show_version(), - ReplCommand::Unknown(msg) => { - eprintln!("{}", msg); - Ok(()) - } + ReplCommand::Invalid(msg) => Err(ReplError::Command(msg)), } } @@ -430,65 +760,45 @@ impl<'a> Repl<'a> { None }; - let result = match self.executor.execute_sql(sql) { - Ok(result) => result, - Err(err) => return Err(ReplError::SqlExecutor(err)), - }; + let result = self.executor.execute(sql)?; - // Print every statement's result set, each with its own header. - for result_set in &result { - if result_set.rows.is_empty() { - println!("Query returned no rows"); - } else { - println!("Query returned {} rows", result_set.rows.len()); - // Header and rows as CSV, quoted where a field needs it - // (NULL is already an empty field in a ResultSet). - let writer = crate::table::RecordWriter { - delimiter: b',', - terminator: b"\n", - comments: false, - }; - let stdout = std::io::stdout(); - let mut out = stdout.lock(); - let _ = writer.write(&mut out, result_set.columns.iter().map(String::as_str)); - for row in &result_set.rows { - let _ = writer.write(&mut out, row.iter().map(String::as_str)); - } - } + // Every statement's result set, each with its own header, printed + // exactly as the command line prints it. + let header = !self.config.no_output_header(); + for table in &result { + let _ = table.print_to_stdout(header); } // The change count is reported independently of whether the line also - // produced rows. Gating it on "no result set" hid the count for a - // trailing DML on a line like `SELECT ...; DELETE ...`. + // produced rows: a trailing DML on `SELECT ...; DELETE ...` reports + // too. It is reported even when zero, so a statement that matched + // nothing still shows that it ran. if self.show_changes && self.executor.last_statement_changed_rows() { - // For non-SELECT statements that don't return rows (INSERT, UPDATE, DELETE) - // Try to display the number of affected rows if show_changes is enabled - // Reported even when zero. With `.changes on` a statement that - // matched nothing otherwise produced no output at all, leaving it - // ambiguous whether it had run. if let Ok(affected_rows) = self.executor.get_affected_row_count() { - println!("{} rows affected", affected_rows); + outln!("{} rows affected", affected_rows); } } // Display query statistics if enabled if let Some(start_time) = start_time { let execution_time = start_time.elapsed(); - println!("Run Time: {:.3} ms", execution_time.as_secs_f64() * 1000.0); + outln!("Run Time: {:.3} ms", execution_time.as_secs_f64() * 1000.0); } - // Save changes if write mode is enabled - if self.config.write_changes() { - let saved_count = match self.executor.save_modified_tables() { - Ok(count) => count, - Err(err) => return Err(ReplError::SqlExecutor(err)), - }; + // Save changes if write mode is enabled. Inside a transaction nothing + // is saved until COMMIT: uncommitted changes never reach a file. + if !self.executor.in_transaction() { + if self.config.write_changes() { + let saved_count = self.executor.save_modified_tables()?; - if saved_count > 0 { - println!("Changes saved to {} tables", saved_count); + if saved_count > 0 { + self.note(format_args!("Changes saved to {} tables", saved_count)); + } + } else if self.executor.has_modified_tables() { + self.note(format_args!( + "Changes not saved: use .write to save changes to files" + )); } - } else if self.executor.has_modified_tables() { - println!("Changes not saved: use .write to save changes to files"); } Ok(()) @@ -496,99 +806,71 @@ impl<'a> Repl<'a> { /// Load a file into a table fn load_file(&mut self, file_spec: &str) -> Result<()> { - // The file load now needs to pass the field separator - // Since our SqlExecutor's load_file method has been updated to handle it - let result = match self.executor.load_file(file_spec) { - Ok(result) => result, - Err(err) => return Err(ReplError::Sqawk(err)), - }; - - match result { + match self.executor.load_file(file_spec)? { Some((table_name, file_path)) => { - println!("Loaded table '{}' from '{}'", table_name, file_path); - Ok(()) - } - None => { - println!("No table created"); - Ok(()) + self.note(format_args!( + "Loaded table '{}' from '{}'", + table_name, file_path + )); } + None => self.note(format_args!("No table created")), } + Ok(()) } - /// Show the list of tables, optionally filtered by a pattern + /// Show the list of tables, optionally only those whose name matches the + /// LIKE pattern `pattern` (`%` and `_` wildcards, ASCII case-insensitive). fn show_tables(&self, pattern: Option<&str>) -> Result<()> { let tables = self.executor.table_names(); if tables.is_empty() { - println!("No tables loaded"); + outln!("No tables loaded"); return Ok(()); } - println!("Tables:"); - match pattern { - Some(pat) => { - // Filter tables matching the pattern (SQL LIKE pattern) - // Convert SQL LIKE pattern to regex - let regex_pattern = pat.replace("%", ".*").replace("_", "."); - let regex = regex::Regex::new(&format!("^{}$", regex_pattern)) - .unwrap_or_else(|_| regex::Regex::new(".*").unwrap()); // Fallback to match all if regex is invalid - - let matching_tables: Vec<&String> = - tables.iter().filter(|name| regex.is_match(name)).collect(); - - if matching_tables.is_empty() { - println!(" No tables match pattern: {}", pat); - } else { - for table in matching_tables { - let modified = if self.executor.is_table_modified(table) { - " (modified)" - } else { - "" - }; - println!(" {}{}", table, modified); - } - } - } - None => { - // Show all tables - for table in tables { - let modified = if self.executor.is_table_modified(&table) { - " (modified)" - } else { - "" - }; - println!(" {}{}", table, modified); - } - } + outln!("Tables:"); + let like = pattern.map(|p| Pattern::new(p, None, Case::Ascii)); + let matching: Vec<&String> = tables + .iter() + .filter(|name| like.as_ref().is_none_or(|l| l.matches(name))) + .collect(); + if matching.is_empty() { + outln!(" No tables match pattern: {}", pattern.unwrap_or("")); + } + for table in matching { + let modified = if self.executor.is_table_modified(table) { + " (modified)" + } else { + "" + }; + outln!(" {}{}", table, modified); } Ok(()) } - // The show_columns functionality is now handled by show_schema with a specific table name - /// Show help message fn show_help(&self) -> Result<()> { - println!("Available commands:"); - println!(" .cd DIRECTORY Change the working directory to DIRECTORY"); - println!( + outln!("Available commands:"); + outln!(" .cd DIRECTORY Change the working directory to DIRECTORY"); + outln!( " .changes [on|off] Show number of rows changed by SQL (currently: {})", if self.show_changes { "ON" } else { "OFF" } ); - println!(" .exit ?CODE? Exit the REPL with optional code"); - println!(" .help Show this help message"); - println!(" .load [TABLE=]FILE Load FILE into TABLE"); - println!(" .print STRING... Print literal STRING"); - println!(" .quit Exit the REPL"); - println!(" .save ?TABLE? Save changes to all tables or a specific TABLE"); - println!(" .schema ?TABLE? Show schema for a specific table or all tables"); - println!(" .show ?WHAT? Show current settings and status information"); - println!( + outln!(" .exit ?CODE? Exit the REPL with exit status CODE"); + outln!(" .help Show this help message"); + outln!(" .load [TABLE=]FILE Load FILE into TABLE"); + outln!(" .print STRING... Print literal STRING"); + outln!(" .quit Exit the REPL"); + outln!(" .save ?TABLE? Save changes to all tables or a specific TABLE"); + outln!(" .schema ?TABLE? Show schema for a specific table or all tables"); + outln!(" .show ?WHAT? Show current settings and status information"); + outln!( " .stats [on|off] Toggle statistics display (currently: {})", if self.show_stats { "ON" } else { "OFF" } ); - println!(" .tables ?TABLE? List names of tables matching LIKE pattern TABLE"); - println!(" .version Show source, library and compiler versions"); - println!( + outln!(" .tables ?PATTERN? List names of tables matching LIKE pattern PATTERN"); + outln!(" .version Show source, library and compiler versions"); + outln!( " .write [on|off] Toggle writing changes to files (currently: {})", if self.config.write_changes() { "ON" @@ -596,120 +878,85 @@ impl<'a> Repl<'a> { "OFF" } ); - println!(" SQL_STATEMENT Execute SQL statement"); + outln!(" SQL_STATEMENT Execute SQL statement"); Ok(()) } - /// Exit the REPL with an optional exit code + /// Exit the REPL, with exit status `code` when given. The status is + /// taken modulo 256, as the operating system reports it. fn exit_repl(&mut self, code: Option<&str>) -> Result<()> { + if let Some(code_str) = code.filter(|c| !c.is_empty()) { + let code = code_str + .parse::() + .map_err(|_| ReplError::Command(format!("invalid exit code: {}", code_str)))?; + self.exit_code = Some(code.rem_euclid(256) as u8); + } self.running = false; + Ok(()) + } - // If an exit code is provided, we'll just acknowledge it - // In a real program, this would set the process exit code - if let Some(code_str) = code { - match code_str.parse::() { - Ok(code) => { - println!("Exit code set to: {}", code); - } - Err(_) => { - eprintln!("Invalid exit code: {}", code_str); - } - } + /// Print the CREATE TABLE form of `name`'s columns. + fn print_schema(&self, name: &str) -> Result<()> { + let column_types = self.executor.get_table_column_types(name)?; + outln!("CREATE TABLE {} (", name); + for (i, (column_name, data_type)) in column_types.iter().enumerate() { + let sep = if i + 1 < column_types.len() { "," } else { "" }; + outln!(" {} {}{}", column_name, data_type, sep); } - + outln!(");"); Ok(()) } /// Display schema information for a table or all tables fn show_schema(&self, table_name: Option<&str>) -> Result<()> { match table_name { - Some(name) => { - // Show schema for specific table - match self.executor.get_table_column_types(name) { - Ok(column_types) => { - println!("CREATE TABLE {} (", name); - for (i, (column_name, data_type)) in column_types.iter().enumerate() { - if i < column_types.len() - 1 { - println!(" {} {},", column_name, data_type); - } else { - println!(" {} {}", column_name, data_type); - } - } - println!(");"); - } - Err(_) => { - eprintln!("No such table: {}", name); - } - } - } + Some(name) => self.print_schema(name), None => { - // Show schema for all tables for name in self.executor.table_names() { - if let Ok(column_types) = self.executor.get_table_column_types(&name) { - println!("CREATE TABLE {} (", name); - for (i, (column_name, data_type)) in column_types.iter().enumerate() { - if i < column_types.len() - 1 { - println!(" {} {},", column_name, data_type); - } else { - println!(" {} {}", column_name, data_type); - } - } - println!(");"); - } + self.print_schema(&name)?; } + Ok(()) } } - Ok(()) } /// Change the current working directory fn change_directory(&self, dir: &str) -> Result<()> { - match std::env::set_current_dir(dir) { - Ok(_) => { - println!("Changed directory to {}", dir); - Ok(()) - } - Err(e) => { - eprintln!("Failed to change directory: {}", e); - Ok(()) - } - } + std::env::set_current_dir(dir).map_err(|e| { + ReplError::Command(format!( + "cannot change directory to '{}': {}", + dir, + crate::error::io_reason(&e) + )) + })?; + self.note(format_args!("Changed directory to {}", dir)); + Ok(()) } /// Toggle showing number of rows changed by SQL statements fn toggle_changes(&mut self, arg: Option<&str>) -> Result<()> { - match arg { - Some("on") => { - self.show_changes = true; - println!("Changes display enabled"); - } - Some("off") => { - self.show_changes = false; - println!("Changes display disabled"); - } - _ => { - // Toggle current state - self.show_changes = !self.show_changes; - println!( - "Changes display {}", - if self.show_changes { - "enabled" - } else { - "disabled" - } - ); + self.show_changes = match arg { + Some("on") => true, + Some("off") => false, + _ => !self.show_changes, + }; + self.note(format_args!( + "Changes display {}", + if self.show_changes { + "enabled" + } else { + "disabled" } - } + )); Ok(()) } /// Show version information /// /// Read from the manifest rather than written out, so it cannot drift. - /// This reported 0.1.1 for a long time while the crate was at 0.8.0. fn show_version(&self) -> Result<()> { - println!("Sqawk version {}", env!("CARGO_PKG_VERSION")); - println!("sqlparser {}", crate::vm::SQLPARSER_VERSION); + outln!("Sqawk version {}", env!("CARGO_PKG_VERSION")); + outln!("sqlparser {}", crate::vm::SQLPARSER_VERSION); Ok(()) } @@ -719,6 +966,11 @@ impl<'a> Repl<'a> { /// if one is specified. This is useful when write mode is off but you want to /// save specific changes. fn save_tables(&mut self, table_name: Option<&str>) -> Result<()> { + if self.executor.in_transaction() { + return Err(ReplError::Sqawk(crate::error::SqawkError::InvalidSqlQuery( + "cannot save within a transaction: COMMIT or ROLLBACK first".to_string(), + ))); + } match table_name { Some(name) => { // Save a specific table if it exists and is modified @@ -729,79 +981,40 @@ impl<'a> Repl<'a> { } if !self.executor.table_is_modified(name) { - println!("Table '{}' has no changes to save", name); + self.note(format_args!("Table '{}' has no changes to save", name)); return Ok(()); } - match self.executor.save_table(name) { - Ok(_) => { - println!("Changes saved to table '{}'", name); - Ok(()) - } - Err(err) => { - // Print specific guidance for NoFilePath errors related to CREATE TABLE usage - if let crate::error::SqawkError::NoFilePath(table) = &err { - eprintln!("Error: Table '{}' has no associated file path", table); - eprintln!("Hint: When creating tables with CREATE TABLE, use the LOCATION clause"); - eprintln!( - "Example: CREATE TABLE {} (...) LOCATION './file.csv';", - table - ); - } else { - eprintln!("Error saving table '{}': {}", name, err); - } - Err(ReplError::Sqawk(err)) - } - } + self.executor.save_table(name)?; + self.note(format_args!("Changes saved to table '{}'", name)); + Ok(()) } None => { // Save all modified tables if !self.executor.has_modified_tables() { - println!("No modified tables to save"); + self.note(format_args!("No modified tables to save")); return Ok(()); } - match self.executor.save_modified_tables() { - Ok(count) => { - println!("Changes saved to {} tables", count); - Ok(()) - } - Err(err) => { - eprintln!("Error saving modified tables: {}", err); - Err(ReplError::SqlExecutor(err)) - } - } + let count = self.executor.save_modified_tables()?; + self.note(format_args!("Changes saved to {} tables", count)); + Ok(()) } } } -} -impl Repl<'_> { /// Toggle writing changes to files fn toggle_write(&mut self, arg: Option<&str>) -> Result<()> { - match arg { - Some("on") => { - self.config.set_write_changes(true); - println!("Write mode enabled - changes will be saved to files"); - } - Some("off") => { - self.config.set_write_changes(false); - println!("Write mode disabled - changes will not be saved to files"); - } - _ => { - // Toggle current state - let current = self.config.write_changes(); - self.config.set_write_changes(!current); - println!( - "Write mode {}", - if self.config.write_changes() { - "enabled" - } else { - "disabled" - } - ); - } - } + let on = match arg { + Some("on") => true, + Some("off") => false, + _ => !self.config.write_changes(), + }; + self.config.set_write_changes(on); + self.note(format_args!( + "Write mode {}", + if on { "enabled" } else { "disabled" } + )); Ok(()) } @@ -814,8 +1027,8 @@ impl Repl<'_> { } _ => { // Show general settings - println!("Sqawk Settings:"); - println!( + outln!("Sqawk Settings:"); + outln!( " Write Mode: {}", if self.config.write_changes() { "ON" @@ -823,24 +1036,24 @@ impl Repl<'_> { "OFF" } ); - println!( + outln!( " Changes Display: {}", if self.show_changes { "ON" } else { "OFF" } ); - println!( + outln!( " Statistics: {}", if self.show_stats { "ON" } else { "OFF" } ); - println!( + outln!( " Verbose: {}", if self.config.verbose() { "ON" } else { "OFF" } ); // Show field separator if defined if let Some(sep) = self.config.field_separator() { - println!(" Field Separator: '{}'", sep); + outln!(" Field Separator: '{}'", sep); } else { - println!(" Field Separator: Default (auto-detect)"); + outln!(" Field Separator: Default (tab for .tsv/.tab, else comma)"); } // Show counts @@ -850,8 +1063,8 @@ impl Repl<'_> { .filter(|t| self.executor.is_table_modified(t)) .count(); - println!(" Tables Loaded: {}", tables.len()); - println!(" Modified Tables: {}", modified_count); + outln!(" Tables Loaded: {}", tables.len()); + outln!(" Modified Tables: {}", modified_count); Ok(()) } @@ -862,29 +1075,25 @@ impl Repl<'_> { fn show_tables_metadata(&self) -> Result<()> { let tables = self.executor.table_names(); if tables.is_empty() { - println!("No tables loaded"); + outln!("No tables loaded"); return Ok(()); } - println!("Table Information:"); + outln!("Table Information:"); for table_name in tables { - // Get table metadata let is_modified = self.executor.is_table_modified(&table_name); let column_count = match self.executor.get_table_columns(&table_name) { Ok(cols) => cols.len(), Err(_) => 0, }; - println!(" Table: {}", table_name); - println!( + outln!(" Table: {}", table_name); + outln!( " Status: {}", if is_modified { "MODIFIED" } else { "Unchanged" } ); - println!(" Columns: {}", column_count); - // We don't have direct access to row count or filename in current implementation - // These would be good additions to the API in the future - - println!(); + outln!(" Columns: {}", column_count); + outln!(); } Ok(()) @@ -892,35 +1101,196 @@ impl Repl<'_> { /// Show statistics or toggle statistics mode fn show_stats(&mut self, option: Option<&str>) -> Result<()> { - match option { - Some("on") => { - self.show_stats = true; - println!("Statistics display enabled"); - Ok(()) - } - Some("off") => { - self.show_stats = false; - println!("Statistics display disabled"); - Ok(()) + self.show_stats = match option { + Some("on") => true, + Some("off") => false, + None => !self.show_stats, + Some(other) => { + return Err(ReplError::Command(format!( + "unknown option for .stats: {} (usage: .stats [on|off])", + other + ))) } - None => { - // Toggle the current state - self.show_stats = !self.show_stats; - println!( - "Statistics display {}", - if self.show_stats { - "enabled" - } else { - "disabled" - } - ); - Ok(()) - } - Some(_) => { - println!("Unknown option for .stats"); - println!("Usage: .stats [on|off]"); - Ok(()) + }; + self.note(format_args!( + "Statistics display {}", + if self.show_stats { + "enabled" + } else { + "disabled" } - } + )); + Ok(()) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn schema() -> Schema { + vec![ + ( + "employees".to_string(), + vec!["id".to_string(), "name".to_string(), "dept".to_string()], + ), + ( + "Depts".to_string(), + vec!["dept".to_string(), "Descr".to_string()], + ), + ] + } + + fn complete(line: &str) -> (usize, Vec) { + complete_line(line, line.len(), &schema()) + } + + #[test] + fn completes_dot_commands() { + assert_eq!(complete(".ta"), (0, vec![".tables".to_string()])); + assert_eq!( + complete(".s"), + ( + 0, + vec![ + ".save".to_string(), + ".schema".to_string(), + ".show".to_string(), + ".stats".to_string() + ] + ) + ); + assert_eq!(complete(" .VER"), (2, vec![".version".to_string()])); + } + + #[test] + fn completes_table_argument_of_dot_commands() { + assert_eq!(complete(".schema em"), (8, vec!["employees".to_string()])); + assert_eq!(complete(".save d"), (6, vec!["Depts".to_string()])); + assert_eq!(complete(".cd em"), (6, vec![])); + } + + #[test] + fn completes_keywords_in_typed_case() { + assert_eq!(complete("SEL"), (0, vec!["SELECT".to_string()])); + assert_eq!(complete("sel"), (0, vec!["select".to_string()])); + let (start, cands) = complete("SELECT * FROM employees WH"); + assert_eq!(start, 24); + assert_eq!(cands, vec!["WHEN".to_string(), "WHERE".to_string()]); + } + + #[test] + fn completes_function_names() { + let (_, cands) = complete("SELECT upp"); + assert_eq!(cands, vec!["upper".to_string()]); + let (_, cands) = complete("SELECT ROW_"); + assert_eq!(cands, vec!["ROW_NUMBER".to_string()]); + } + + #[test] + fn completes_table_and_column_names_case_insensitively() { + let (start, cands) = complete("SELECT * FROM EMP"); + assert_eq!(start, 14); + assert_eq!(cands, vec!["employees".to_string()]); + // Tables and columns first, then keywords; duplicates folded. + let (_, cands) = complete("SELECT de"); + assert_eq!( + cands, + vec![ + "Depts".to_string(), + "dept".to_string(), + "Descr".to_string(), + "delete".to_string(), + "dense_rank".to_string(), + "desc".to_string() + ] + ); + } + + #[test] + fn completes_columns_after_table_qualifier() { + assert_eq!( + complete("SELECT employees.n"), + (17, vec!["name".to_string()]) + ); + assert_eq!( + complete("SELECT DEPTS."), + (13, vec!["dept".to_string(), "Descr".to_string()]) + ); + assert_eq!(complete("SELECT nosuch.x"), (14, vec![])); + } + + #[test] + fn empty_word_completes_table_names() { + assert_eq!( + complete("SELECT * FROM "), + (14, vec!["employees".to_string(), "Depts".to_string()]) + ); + } + + #[test] + fn completion_respects_cursor_position() { + let line = "SELECT na FROM employees"; + assert_eq!( + complete_line(line, 9, &schema()), + (7, vec!["name".to_string(), "natural".to_string()]) + ); + } + + #[test] + fn completer_trait_returns_pairs() { + let helper = ReplHelper { schema: schema() }; + let history = DefaultHistory::new(); + let ctx = Context::new(&history); + let (start, pairs) = helper.complete("SELECT * FROM emp", 17, &ctx).unwrap(); + assert_eq!(start, 14); + assert_eq!(pairs.len(), 1); + assert_eq!(pairs[0].replacement, "employees"); + } + + #[test] + fn scan_finds_statement_end() { + let s = |t: &str| { + let r = scan_sql(t); + (r.has_tokens, r.ends_with_semicolon) + }; + assert_eq!(s(""), (false, false)); + assert_eq!(s(" -- only a comment\n"), (false, false)); + assert_eq!(s("/* c */ ;"), (false, true)); + assert_eq!(s("SELECT 1"), (true, false)); + assert_eq!(s("SELECT 1;"), (true, true)); + assert_eq!(s("SELECT 1; -- trailing"), (true, true)); + assert_eq!(s("SELECT 1; /* x */"), (true, true)); + assert_eq!(s("SELECT ';"), (true, false)); + assert_eq!(s("SELECT ';'"), (true, false)); + assert_eq!(s("SELECT 'it''s';"), (true, true)); + assert_eq!(s("SELECT \"a;\" FROM t"), (true, false)); + assert_eq!(s("SELECT 1; SELECT 2"), (true, false)); + } + + #[test] + fn history_location() { + // Absolute paths for this platform: "/state" has no drive letter and + // is relative on Windows. + #[cfg(windows)] + let (state, home) = (r"C:\state", r"C:\home\u"); + #[cfg(not(windows))] + let (state, home) = ("/state", "/home/u"); + let os = |s: &str| Some(OsString::from(s)); + assert_eq!( + history_path(os(state), os(home)), + Some(PathBuf::from(state).join("sqawk").join("history")) + ); + assert_eq!( + history_path(None, os(home)), + Some(PathBuf::from(home).join(".sqawk_history")) + ); + // A relative XDG_STATE_HOME is invalid and ignored. + assert_eq!( + history_path(os("rel"), os(home)), + Some(PathBuf::from(home).join(".sqawk_history")) + ); + assert_eq!(history_path(os(""), os("")), None); + assert_eq!(history_path(None, None), None); } } diff --git a/src/sql_executor.rs b/src/sql_executor.rs index b54dda5..dc7320f 100644 --- a/src/sql_executor.rs +++ b/src/sql_executor.rs @@ -10,30 +10,21 @@ //! The module implements a non-destructive approach, modifying only in-memory tables //! until explicitly requested to save changes back to the original files. -use std::collections::HashSet; - -use anyhow::Result; - -use crate::capacity::DEFAULT_TABLE_CAPACITY; use crate::config::AppConfig; use crate::database::Database; use crate::error::{SqawkError, SqawkResult}; use crate::file_handler::FileHandler; -use crate::table::DataType; /// SQL statement executor /// /// This executor wraps the VM execution engine and provides additional functionality /// for tracking modified tables and integrating with the REPL interface. -pub struct SqlExecutor<'a> { +pub struct SqlExecutor { /// Database for storing and accessing tables - database: &'a mut Database, + database: Database, /// File handler for loading and saving tables - file_handler: &'a mut FileHandler, - - /// Names of tables that have been modified - modified_tables: HashSet, + file_handler: FileHandler, /// Application configuration for global settings config: AppConfig, @@ -44,17 +35,13 @@ pub struct SqlExecutor<'a> { dml_executed: bool, } -impl<'a> SqlExecutor<'a> { - /// Create a new SQL executor with the given database, file handler, and application configuration - pub fn new( - database: &'a mut Database, - file_handler: &'a mut FileHandler, - config: &AppConfig, - ) -> Self { +impl SqlExecutor { + /// Create a new SQL executor that owns the given database and file + /// handler, with the application configuration + pub fn new(database: Database, file_handler: FileHandler, config: &AppConfig) -> Self { SqlExecutor { database, file_handler, - modified_tables: HashSet::with_capacity(DEFAULT_TABLE_CAPACITY), config: config.clone(), affected_rows: 0, dml_executed: false, @@ -77,27 +64,22 @@ impl<'a> SqlExecutor<'a> { /// deliberately NOT merged, since each has its own column schema. pub fn execute(&mut self, sql: &str) -> SqawkResult> { if self.config.verbose() { - eprintln!("Executing SQL: {}", sql); + diag!("Executing SQL: {}", sql); } - // Execute via VM engine - let result = crate::vm::execute_vm(sql, self.database, self.config.verbose())?; - - // Track modified tables - for table_name in result.modified_tables { - self.modified_tables.insert(table_name); - } + // Execute via VM engine (which also records the modified tables in + // the database) + let result = crate::vm::execute_vm(sql, &mut self.database, self.config.verbose())?; // Track affected rows from the last statement self.affected_rows = result.affected_rows; self.dml_executed = result.dml_executed; - // Set delimiter on each result table to match config + // Result tables print with the output delimiter let mut tables = result.tables; - if let Some(delim) = self.config.field_separator() { - for t in &mut tables { - t.set_delimiter(delim.clone()); - } + let delim = self.config.output_delimiter(); + for t in &mut tables { + t.set_delimiter(delim.clone()); } Ok(tables) @@ -105,35 +87,59 @@ impl<'a> SqlExecutor<'a> { /// Save all modified tables back to their source files /// - /// This function writes any tables that have been modified during execution - /// (e.g., through INSERT, UPDATE, or DELETE statements) back to their source files. - /// Only tables that have been modified will be saved, preserving the original - /// files if no changes were made. + /// Writes every table changed by INSERT, UPDATE, DELETE, ALTER TABLE, + /// TRUNCATE or CREATE TABLE ... LOCATION back to its file. A scratch + /// table (no file: CREATE TABLE without LOCATION, CREATE TABLE ... AS) + /// is skipped. /// - /// Every table is checked for a writeback target before any of them is - /// written. An unwritable table (one read from standard input, say) used - /// to surface on its own turn in the loop, after an arbitrary number of - /// real source files had already been rewritten -- and since - /// `modified_tables` is a `HashSet`, which ones varied between runs of the - /// same command. Sorting makes the writes themselves ordered too. + /// Every table is checked before any of them is written: a table read + /// from standard input, a read-only file, or a directory that cannot + /// take the temp file fails the whole writeback with nothing written. + /// Writes happen in table-name order. /// /// # Returns /// * `Ok(usize)` - Number of tables saved /// * `Err` if any error occurs during saving - pub fn save_modified_tables(&self) -> Result { - let mut names: Vec<&String> = self.modified_tables.iter().collect(); - names.sort(); - - for table_name in &names { - self.file_handler.check_table_writable(table_name)?; + pub fn save_modified_tables(&self) -> SqawkResult { + let mut targets = Vec::new(); + for table_name in self.database.modified_tables() { + if self + .file_handler + .check_table_writable(&self.database, &table_name)? + { + targets.push(table_name); + } else if self.config.verbose() { + diag!( + "Table '{}' has no file (CREATE TABLE without LOCATION); not written", + table_name + ); + } } - for table_name in &names { + for table_name in &targets { // Use the file handler to write the table back to its source file - self.file_handler.save_table(table_name)?; + self.file_handler.save_table(&self.database, table_name)?; } - Ok(names.len()) + Ok(targets.len()) + } + + /// Whether a transaction (BEGIN without COMMIT / ROLLBACK yet) is open. + pub fn in_transaction(&self) -> bool { + self.database.in_transaction() + } + + /// End of input: roll back a transaction left open, as SQLite discards + /// uncommitted work when the connection closes. Called before the final + /// writeback, so --write never saves uncommitted changes. + pub fn finish(&mut self) { + if self.database.in_transaction() { + if self.config.verbose() { + diag!("Transaction still open at exit: rolled back"); + } + // Cannot fail: a transaction is open. + let _ = self.database.rollback(); + } } /// Check if a specific table has been modified @@ -144,7 +150,7 @@ impl<'a> SqlExecutor<'a> { /// # Returns /// * `bool` - True if the table has been modified pub fn is_table_modified(&self, table_name: &str) -> bool { - self.modified_tables.contains(table_name) + self.database.is_modified(table_name) } /// Get a list of all available table names @@ -173,13 +179,14 @@ impl<'a> SqlExecutor<'a> { /// * `table_name` - The name of the table /// /// # Returns - /// * `SqawkResult>` - List of column names with their data types - pub fn get_table_column_types(&self, table_name: &str) -> SqawkResult> { + /// * `SqawkResult>` - each column's name and type + /// name (the declared name for a declared column) + pub fn get_table_column_types(&self, table_name: &str) -> SqawkResult> { let table = self.database.get_table(table_name)?; let column_types = table .column_metadata() .iter() - .map(|col| (col.name.clone(), col.data_type)) + .map(|col| (col.name.clone(), col.type_name())) .collect(); Ok(column_types) } @@ -189,7 +196,7 @@ impl<'a> SqlExecutor<'a> { /// # Returns /// * `bool` - True if any tables have been modified pub fn has_modified_tables(&self) -> bool { - !self.modified_tables.is_empty() + !self.database.modified_tables().is_empty() } /// Load a file as a table @@ -199,31 +206,16 @@ impl<'a> SqlExecutor<'a> { /// /// # Returns /// * `SqawkResult>` - Tuple of (table_name, file_path) if successful - pub fn load_file(&mut self, file_spec: &str) -> SqawkResult> { - self.file_handler.load_file(file_spec) - } - - /// Execute SQL statement and return a ResultSet for REPL mode - /// - /// # Arguments - /// * `sql` - SQL statement to execute - /// - /// # Returns - /// * One ResultSet per statement that produced rows, in order. /// - /// Returning only the last one hid earlier result sets AND suppressed the - /// change count of a trailing DML statement, because the caller treated - /// "there is a result set" as "this line produced no changes". - pub fn execute_sql(&mut self, sql: &str) -> Result> { - let result = self.execute(sql)?; - - Ok(result - .iter() - .map(|table| ResultSet { - columns: table.columns().to_vec(), - rows: table.rows_as_strings(), - }) - .collect()) + /// Refused inside a transaction: ROLLBACK restores the tables as they + /// were at BEGIN, which would silently unload the file. + pub fn load_file(&mut self, file_spec: &str) -> SqawkResult> { + if self.database.in_transaction() { + return Err(SqawkError::InvalidSqlQuery( + "cannot load a file within a transaction".to_string(), + )); + } + self.file_handler.load_file(&mut self.database, file_spec) } /// Whether the last executed script ran a row-counting DML statement. @@ -242,7 +234,7 @@ impl<'a> SqlExecutor<'a> { /// # Returns /// * `bool` - True if the table exists pub fn table_exists(&self, table_name: &str) -> bool { - self.file_handler.has_table(table_name) + self.database.has_table(table_name) } /// Check if a table is modified (alias for is_table_modified) @@ -253,7 +245,7 @@ impl<'a> SqlExecutor<'a> { /// # Returns /// * `bool` - True if the table has been modified pub fn table_is_modified(&self, table_name: &str) -> bool { - self.modified_tables.contains(table_name) + self.database.is_modified(table_name) } /// Save a specific table @@ -264,16 +256,16 @@ impl<'a> SqlExecutor<'a> { /// # Returns /// * `SqawkResult<()>` - Success or error pub fn save_table(&self, table_name: &str) -> SqawkResult<()> { - if !self.file_handler.has_table(table_name) { + if !self.database.has_table(table_name) { return Err(SqawkError::TableNotFound(table_name.to_string())); } - if !self.modified_tables.contains(table_name) { + if !self.database.is_modified(table_name) { // Table exists but isn't modified, just return success return Ok(()); } - self.file_handler.save_table(table_name) + self.file_handler.save_table(&self.database, table_name) } /// Get the number of rows affected by the last executed statement @@ -283,12 +275,3 @@ impl<'a> SqlExecutor<'a> { Ok(self.affected_rows) } } - -/// Result set structure for REPL output -#[derive(Debug)] -pub struct ResultSet { - /// Column names - pub columns: Vec, - /// Rows as strings - pub rows: Vec>, -} diff --git a/src/table.rs b/src/table.rs index e5d21dd..1963869 100644 --- a/src/table.rs +++ b/src/table.rs @@ -13,6 +13,7 @@ use std::borrow::Cow; use std::collections::{BTreeMap, HashMap}; use std::fmt; use std::path::PathBuf; +use std::sync::OnceLock; use crate::error::{SqawkError, SqawkResult}; @@ -155,7 +156,7 @@ impl fmt::Display for Value { match self { Value::Null => write!(f, "NULL"), Value::Integer(i) => write!(f, "{}", i), - Value::Float(float) => write!(f, "{}", float), + Value::Float(float) => f.write_str(&crate::vm::arith::real_text(*float)), Value::String(s) => write!(f, "{}", s), Value::Boolean(b) => write!(f, "{}", b), } @@ -182,61 +183,81 @@ fn canonical_integer(s: &str) -> Option { s.parse::().ok() } -/// Parse `s` as a float only if formatting the float back (the same `{}` -/// formatting `Value`'s Display uses) reproduces `s` exactly. +/// Parse `s` as a float only if rendering the float back (the rendering +/// `Value`'s Display uses, SQLite's `CAST(x AS TEXT)`) reproduces `s` +/// exactly. /// -/// So `1.5` is a float but `1.50`, `1e3`, `2.0`, `.5`, `NaN` and `inf` are +/// So `1.5`, `2.0` and `1.0e+20` are floats but `1.50`, `2`, `1e3`, `.5`, +/// `0.30000000000000004` (17 digits; rendered `0.3`), `NaN` and `Inf` are /// not: loading them as numbers would change their spelling on output and on -/// writeback. Display never emits an exponent, so only `-`, digits and `.` -/// can possibly round-trip; anything else is rejected before formatting. +/// writeback. Every rendering contains a point and only digits, `-`, `+`, +/// `.` and `e`; anything else is rejected before parsing. fn canonical_float(s: &str) -> Option { - if s.is_empty() + if !s.contains('.') || !s .bytes() - .all(|c| c.is_ascii_digit() || c == b'-' || c == b'.') + .all(|c| c.is_ascii_digit() || matches!(c, b'-' | b'.' | b'e' | b'+')) { return None; } - let f = s.parse::().ok()?; + s.parse::().ok()?; + let f = crate::vm::arith::atof(s); if !f.is_finite() { return None; } if let Some(canonical) = short_decimal_is_canonical(s) { return canonical.then_some(f); } - thread_local! { - static BUF: std::cell::RefCell = const { std::cell::RefCell::new(String::new()) }; - } - BUF.with(|buf| { - use std::fmt::Write; - let mut buf = buf.borrow_mut(); - buf.clear(); - let _ = write!(buf, "{}", f); - (*buf == s).then_some(f) - }) + (crate::vm::arith::real_text(f) == s).then_some(f) } -/// Decide canonicity of a plain decimal without formatting, when possible. +/// Decide canonicity of a plain decimal without rendering, when possible. /// -/// Display prints the shortest digit string that round-trips, in positional -/// notation. A decimal of at most 15 significant digits round-trips through -/// f64 exactly, and no other decimal that short maps to the same double, so -/// such a string is canonical iff it is in Display's shape: -/// `-?(0|[1-9][0-9]*)\.[0-9]*[1-9]`. Returns None when the fast rule does not -/// apply (more digits, or no point), leaving the caller to format and compare. +/// The rendering is `%.15g` with `.0` kept on integral values: positional +/// when the leading digit's exponent is in [-4, 15), trailing zeros dropped +/// except the one in `.0`. A decimal of at most 15 significant digits comes +/// back unchanged from 15-digit rounding of its nearest double, so such a +/// string is canonical iff it has the rendering's shape. Returns None when +/// the fast rule does not apply (more digits, or exponent notation), leaving +/// the caller to render and compare. fn short_decimal_is_canonical(s: &str) -> Option { - let body = s.strip_prefix('-').unwrap_or(s); + let (neg, body) = match s.strip_prefix('-') { + Some(b) => (true, b), + None => (false, s), + }; + if body.contains('e') { + return None; + } let (int, frac) = body.split_once('.')?; - if int.is_empty() || frac.is_empty() || frac.contains(['.', '-']) || int.contains('-') { + let all_digits = |t: &str| !t.is_empty() && t.bytes().all(|c| c.is_ascii_digit()); + if !all_digits(int) || !all_digits(frac) { return Some(false); } - let digits = || int.bytes().chain(frac.bytes()); - let significant = digits().skip_while(|&c| c == b'0').count(); + if int.len() > 1 && int.starts_with('0') { + return Some(false); + } + let lead_zeros = frac.bytes().take_while(|&c| c == b'0').count(); + if int == "0" && lead_zeros == frac.len() { + // Zero renders `0.0`, negative zero included. + return Some(!neg && frac == "0"); + } + let significant = if int == "0" { + frac.len() - lead_zeros + } else { + int.len() + frac.len() + }; if significant > 15 { return None; } - let shape_ok = (int == "0" || !int.starts_with('0')) && !frac.ends_with('0'); - Some(shape_ok) + let exponent = if int == "0" { + -(lead_zeros as i64) - 1 + } else { + int.len() as i64 - 1 + }; + if !(-4..15).contains(&exponent) { + return Some(false); + } + Some(frac == "0" || !frac.ends_with('0')) } /// Infer the type of a field read from an input file. @@ -246,9 +267,9 @@ fn short_decimal_is_canonical(s: &str) -> Option { /// /// - empty field: NULL (rendered back as an empty field) /// - canonical integer (`42`, `-7`): Integer -/// - canonical float (`1.5`, `-0.25`): Float +/// - canonical float (`1.5`, `-0.25`, `2.0`, `1.0e+20`): Float /// - anything else, verbatim and untrimmed: String. That includes `007`, -/// `1.50`, `1e3`, ` 42`, `true`, `NaN`. +/// `1.50`, `1e3`, `0.30000000000000004`, ` 42`, `true`, `NaN`. /// /// Numeric-looking strings still behave as numbers where a number is needed /// (arithmetic, SUM/AVG, comparison with a number, ordering); see the VM. @@ -302,9 +323,35 @@ impl Value { } } +/// Make `names` unique, case-insensitively, the way SQLite names the +/// columns of a subquery in FROM or a CREATE TABLE ... AS: a name equal to +/// an earlier one gets `:1`, `:2`, ... in place of any `:N` suffix it had, +/// the first that is free (`id, id, id` becomes `id, id:1, id:2`). +pub fn unique_column_names(names: &mut [String]) { + let mut seen: std::collections::HashSet = std::collections::HashSet::new(); + for name in names.iter_mut() { + let mut cnt = 0u32; + while seen.contains(&name.to_lowercase()) { + let digits = name.len() - name.trim_end_matches(|c: char| c.is_ascii_digit()).len(); + let mut base = name.len(); + if digits > 0 && digits < name.len() && name.as_bytes()[name.len() - digits - 1] == b':' + { + base = name.len() - digits - 1; + } + cnt += 1; + *name = format!("{}:{}", &name[..base], cnt); + } + seen.insert(name.to_lowercase()); + } +} + /// Interpret text as a number: an i64 if it parses as one, else a finite f64. +/// +/// This is the single "numeric reading" of text, used by every comparison +/// (see the VM's `sql_compare`), by SUM/AVG and by ordering. Surrounding +/// ASCII whitespace is allowed. pub fn numeric_text(s: &str) -> Option { - let t = s.trim(); + let t = s.trim_matches(|c: char| c.is_ascii_whitespace()); // Cheap rejection first: this runs inside sort comparators. A finite // number starts with a digit, a sign or a point. if !t @@ -317,414 +364,22 @@ pub fn numeric_text(s: &str) -> Option { if let Ok(i) = t.parse::() { return Some(Value::Integer(i)); } - // `inf` and `NaN` parse but are not finite, so they stay text. - match t.parse::() { - Ok(f) if f.is_finite() => Some(Value::Float(f)), - _ => None, - } + // `inf` and `NaN` parse but are not finite, so they stay text. The + // value is SQLite's reading of the text. + t.parse::().ok()?; + Some(crate::vm::arith::atof(t)) + .filter(|f| f.is_finite()) + .map(Value::Float) } /// Represents a row in a table pub type Row = Vec; -// ============================================================================= -// Index-Based Result Types (Phase 4A) -// ============================================================================= -// These types enable efficient query processing by storing row indices instead -// of cloning entire rows. Materialization only happens at output time. -// -// Note: These types are defined in Phase 4A and will be integrated in Phases 4B-4F. -// The #[allow(dead_code)] attributes will be removed as each type is adopted. - -/// Reference to a row within a specific table -/// -/// This lightweight struct (16 bytes) replaces cloning entire rows (~100+ bytes). -/// Used for tracking which rows match query conditions. -#[allow(dead_code)] -#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] -pub struct RowRef { - /// Index of the table in the source table list (for JOINs) - pub table_idx: usize, - /// Index of the row within that table - pub row_idx: usize, -} - -#[allow(dead_code)] -impl RowRef { - /// Create a new row reference - pub fn new(table_idx: usize, row_idx: usize) -> Self { - Self { table_idx, row_idx } - } - - /// Create a row reference for single-table operations (table_idx = 0) - pub fn single(row_idx: usize) -> Self { - Self { - table_idx: 0, - row_idx, - } - } -} - -/// Set of row references for single-table operations -/// -/// Instead of `Vec>` (clones all row data), this stores only -/// indices into the original table. Memory usage: O(n * 8 bytes) vs O(n * row_size). -#[allow(dead_code)] -#[derive(Debug)] -pub struct RowSet<'a> { - /// Reference to the source table (borrowed, not owned) - source: &'a Table, - /// Indices of selected rows (into source.rows) - indices: Vec, -} - -#[allow(dead_code)] -impl<'a> RowSet<'a> { - /// Create a new empty RowSet referencing the given table - pub fn new(source: &'a Table) -> Self { - Self { - source, - indices: Vec::new(), - } - } - - /// Create a RowSet with pre-allocated capacity - pub fn with_capacity(source: &'a Table, capacity: usize) -> Self { - Self { - source, - indices: Vec::with_capacity(capacity), - } - } - - /// Add a row index to the set - pub fn push(&mut self, row_idx: usize) { - self.indices.push(row_idx); - } - - /// Get the number of rows in this set - pub fn len(&self) -> usize { - self.indices.len() - } - - /// Check if the set is empty - pub fn is_empty(&self) -> bool { - self.indices.is_empty() - } - - /// Get the source table - pub fn source(&self) -> &'a Table { - self.source - } - - /// Get the row indices - pub fn indices(&self) -> &[usize] { - &self.indices - } - - /// Get a value from a specific row and column (no clone) - pub fn get_value(&self, row_idx: usize, col_idx: usize) -> Option<&Value> { - let actual_row = *self.indices.get(row_idx)?; - self.source.rows().get(actual_row)?.get(col_idx) - } - - /// Materialize a single row (clones values - use sparingly) - pub fn materialize_row(&self, row_idx: usize) -> Option> { - let actual_row = *self.indices.get(row_idx)?; - self.source.rows().get(actual_row).cloned() - } - - /// Iterate over row indices - pub fn iter(&self) -> impl Iterator + '_ { - self.indices.iter().copied() - } -} - -/// Row references for multi-table operations (JOINs) -/// -/// For JOIN operations, we need to track which rows from each table -/// form a result row. Option handles NULL rows in outer joins. -#[allow(dead_code)] -#[derive(Debug)] -pub struct JoinedRowSet<'a> { - /// References to source tables (borrowed) - sources: Vec<&'a Table>, - /// Each entry is row indices from each source table - /// Option allows NULL rows for outer joins - row_pairs: Vec>>, -} - -#[allow(dead_code)] -impl<'a> JoinedRowSet<'a> { - /// Create a new JoinedRowSet for two tables - pub fn new(left: &'a Table, right: &'a Table) -> Self { - Self { - sources: vec![left, right], - row_pairs: Vec::new(), - } - } - - /// Create with pre-allocated capacity - pub fn with_capacity(left: &'a Table, right: &'a Table, capacity: usize) -> Self { - Self { - sources: vec![left, right], - row_pairs: Vec::with_capacity(capacity), - } - } - - /// Add a matched row pair (INNER JOIN) - pub fn add_match(&mut self, left_idx: usize, right_idx: usize) { - self.row_pairs.push(vec![Some(left_idx), Some(right_idx)]); - } - - /// Add unmatched left row with NULL right (LEFT JOIN) - pub fn add_left_only(&mut self, left_idx: usize) { - self.row_pairs.push(vec![Some(left_idx), None]); - } - - /// Add unmatched right row with NULL left (RIGHT JOIN) - pub fn add_right_only(&mut self, right_idx: usize) { - self.row_pairs.push(vec![None, Some(right_idx)]); - } - - /// Get the number of result rows - pub fn len(&self) -> usize { - self.row_pairs.len() - } - - /// Check if empty - pub fn is_empty(&self) -> bool { - self.row_pairs.is_empty() - } - - /// Get value from a joined row (returns &Value or NULL for outer join misses) - pub fn get_value(&self, row_idx: usize, table_idx: usize, col_idx: usize) -> &Value { - // Static NULL for outer join misses - static NULL_VALUE: Value = Value::Null; - - if let Some(row_pair) = self.row_pairs.get(row_idx) { - if let Some(Some(actual_row)) = row_pair.get(table_idx) { - if let Some(table) = self.sources.get(table_idx) { - if let Some(row) = table.rows().get(*actual_row) { - if let Some(value) = row.get(col_idx) { - return value; - } - } - } - } - } - &NULL_VALUE - } - - /// Get the source tables - pub fn sources(&self) -> &[&'a Table] { - &self.sources - } - - /// Materialize a single result row (clones values) - pub fn materialize_row( - &self, - row_idx: usize, - left_cols: usize, - right_cols: usize, - ) -> Option> { - // Verify row exists - let _ = self.row_pairs.get(row_idx)?; - let mut result = Vec::with_capacity(left_cols + right_cols); - - // Left table columns - for col_idx in 0..left_cols { - result.push(self.get_value(row_idx, 0, col_idx).clone()); - } - - // Right table columns - for col_idx in 0..right_cols { - result.push(self.get_value(row_idx, 1, col_idx).clone()); - } - - Some(result) - } -} - -/// Unified VM result type - either single-table indices, joined indices, or materialized -/// -/// This enum allows the VM to work with index-based results throughout query -/// processing, only materializing to actual values when necessary. -#[allow(dead_code)] -#[derive(Debug)] -pub enum IndexedResult<'a> { - /// Result from single table scan/filter - Single(RowSet<'a>), - /// Result from JOIN operations - Joined(JoinedRowSet<'a>), - /// Materialized result (for complex expressions, aggregates, or final output) - Materialized(Vec>), -} - -#[allow(dead_code)] -impl<'a> IndexedResult<'a> { - /// Get the number of result rows - pub fn len(&self) -> usize { - match self { - IndexedResult::Single(rs) => rs.len(), - IndexedResult::Joined(js) => js.len(), - IndexedResult::Materialized(rows) => rows.len(), - } - } - - /// Check if empty - pub fn is_empty(&self) -> bool { - self.len() == 0 - } - - /// Check if this result is already materialized - pub fn is_materialized(&self) -> bool { - matches!(self, IndexedResult::Materialized(_)) - } -} - -/// Lazy row view - provides access to row values without immediate cloning -/// -/// This struct allows iteration over result rows while deferring value -/// cloning until actually needed (e.g., for output). -#[allow(dead_code)] -#[derive(Debug)] -pub struct LazyRow<'a, 'b> { - /// Reference to the indexed result - result: &'b IndexedResult<'a>, - /// Index within the result - row_idx: usize, -} - -#[allow(dead_code)] -impl<'a, 'b> LazyRow<'a, 'b> { - /// Create a new lazy row view - pub fn new(result: &'b IndexedResult<'a>, row_idx: usize) -> Self { - Self { result, row_idx } - } - - /// Get the row index - pub fn index(&self) -> usize { - self.row_idx - } - - /// Materialize this row to owned values (clones - use when needed for output) - pub fn materialize(&self) -> Option> { - match self.result { - IndexedResult::Single(rs) => rs.materialize_row(self.row_idx), - IndexedResult::Joined(js) => { - // For joined rows, we need column counts - let left_cols = js.sources().first().map(|t| t.column_count()).unwrap_or(0); - let right_cols = js.sources().get(1).map(|t| t.column_count()).unwrap_or(0); - js.materialize_row(self.row_idx, left_cols, right_cols) - } - IndexedResult::Materialized(rows) => rows.get(self.row_idx).cloned(), - } - } -} - -/// Builder for constructing IndexedResult during VM execution -/// -/// This builder accumulates row indices during query processing and -/// produces an IndexedResult at the end. -#[allow(dead_code)] -#[derive(Debug)] -pub struct IndexedResultBuilder<'a> { - /// Source tables being queried - sources: Vec<&'a Table>, - /// Accumulated row indices (single table: one index per row) - single_indices: Vec, - /// Accumulated row pairs (joins: multiple indices per row) - join_indices: Vec>>, - /// Whether this is a join operation - is_join: bool, -} - -#[allow(dead_code)] -impl<'a> IndexedResultBuilder<'a> { - /// Create a new builder - pub fn new() -> Self { - Self { - sources: Vec::new(), - single_indices: Vec::new(), - join_indices: Vec::new(), - is_join: false, - } - } - - /// Add a source table - pub fn add_source(&mut self, table: &'a Table) { - self.sources.push(table); - if self.sources.len() > 1 { - self.is_join = true; - } - } - - /// Add a single row index (for single-table operations) - pub fn add_row_single(&mut self, row_idx: usize) { - self.single_indices.push(row_idx); - } - - /// Add a joined row (for multi-table operations) - pub fn add_row_joined(&mut self, indices: Vec>) { - self.join_indices.push(indices); - } - - /// Check if any rows have been added - pub fn is_empty(&self) -> bool { - self.single_indices.is_empty() && self.join_indices.is_empty() - } - - /// Get the number of result rows - pub fn len(&self) -> usize { - if self.is_join { - self.join_indices.len() - } else { - self.single_indices.len() - } - } - - /// Clear accumulated results (for reuse) - pub fn clear(&mut self) { - self.single_indices.clear(); - self.join_indices.clear(); - } - - /// Build the final IndexedResult - pub fn build(self) -> Option> { - if self.sources.is_empty() { - return None; - } - - if self.is_join && self.sources.len() >= 2 { - let joined = JoinedRowSet { - sources: self.sources, - row_pairs: self.join_indices, - }; - Some(IndexedResult::Joined(joined)) - } else if let Some(source) = self.sources.into_iter().next() { - let row_set = RowSet { - source, - indices: self.single_indices, - }; - Some(IndexedResult::Single(row_set)) - } else { - None - } - } -} - -impl<'a> Default for IndexedResultBuilder<'a> { - fn default() -> Self { - Self::new() - } -} - -// ============================================================================= -// End Index-Based Result Types -// ============================================================================= - /// Represents an in-memory table -#[derive(Debug)] +/// +/// `Clone` is a deep copy (rows, source bytes and layout); a transaction +/// keeps one of each table it changes, for ROLLBACK. +#[derive(Debug, Clone)] pub struct Table { /// Name of the table name: String, @@ -732,9 +387,6 @@ pub struct Table { /// Complete column metadata including name and type cols: Vec, - /// Map of column names to their indices - column_map: HashMap, - /// Row data rows: Vec, @@ -755,15 +407,78 @@ pub struct Table { /// A `--tabledef` schema with no data yet, waiting for its file placeholder: bool, + + /// The source bytes of the header and of each loaded row, so writeback + /// reproduces untouched rows byte-for-byte (see [`RawRows`]). + raw: Option>, + + /// Lower-cased column name -> index, built on the first + /// [`Table::column_index`] of a wide table so that resolving every + /// column of a wide table costs linear, not quadratic, time. Reset + /// whenever the columns change. + name_index: OnceLock>, +} + +/// The bytes each record of a table's source file was read from. +/// +/// The whole file is kept in one buffer and every row points into it by +/// [`RawSpan`], parallel to `Table::rows`. A row whose span is +/// [`RawSpan::NONE`] (inserted, or changed by UPDATE) is written by the +/// [`RecordWriter`]; every other row is written as the exact bytes it was read +/// from, line ending included -- quoting, padding, invalid UTF-8 and all. +#[derive(Debug, Clone)] +pub struct RawRows { + /// The file contents as read (byte order mark included). + pub buf: Vec, + /// The header record's bytes, if the file has a header. + pub header: RawSpan, + /// One span per row of the table. + pub rows: Vec, +} + +/// A byte range `start..end` of [`RawRows::buf`]; `NONE` when there is none. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct RawSpan { + /// First byte of the record + pub start: usize, + /// One past the last byte (the line ending, when there is one) + pub end: usize, +} + +impl RawSpan { + /// No source bytes: the record is written from its values. + pub const NONE: RawSpan = RawSpan { + start: usize::MAX, + end: usize::MAX, + }; + + fn get(self, buf: &[u8]) -> Option<&[u8]> { + buf.get(self.start..self.end) + } +} + +/// Whether two values are the same value of the same type, so a row that +/// UPDATE rewrote with them is unchanged and keeps its source bytes. Stricter +/// than `==`, which equates 1 and 1.0. +fn identical(a: &Value, b: &Value) -> bool { + match (a, b) { + (Value::Null, Value::Null) => true, + (Value::Integer(x), Value::Integer(y)) => x == y, + (Value::Float(x), Value::Float(y)) => x.to_bits() == y.to_bits(), + (Value::String(x), Value::String(y)) => x == y, + (Value::Boolean(x), Value::Boolean(y)) => x == y, + _ => false, + } } /// Layout details of a table's source file that writeback must reproduce. #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub struct FileLayout { - /// Whether the file has a header line. Headerless files (`--no-header`, - /// `--tabledef`, or a guessed headerless non-.csv file) are written back - /// without one. + /// Whether the file has a header line. Headerless files (`--no-header` + /// or `--tabledef`) are written back without one. pub has_header: bool, + /// Whether the file starts with a UTF-8 byte order mark. + pub bom: bool, /// Whether records end in CRLF rather than LF. pub crlf: bool, /// Whether `#` lines are comments when the file is read. If so, a data @@ -772,15 +487,16 @@ pub struct FileLayout { pub comments: bool, } -/// The `#` comment lines of a source file read with comment skipping, each -/// kept verbatim (with its own line ending) and anchored to the data around -/// it, so that writeback puts it back where it was. +/// The lines of a source file between its records -- blank lines, and the +/// `#` comment lines of a file read with `--comments` -- each kept verbatim +/// (with its own line ending) and anchored to the data around it, so that +/// writeback puts it back where it was. /// -/// A comment line is anchored to the row that follows it. Rows keep their -/// identity through UPDATE, so their comments stay with them; a deleted row -/// leaves its comments in place, in front of the next surviving row (or of -/// the trailing comments); rows appended by INSERT go after the last row and -/// before the trailing comments. Blank lines are not kept. +/// A line is anchored to the row that follows it. Rows keep their identity +/// through UPDATE, so their lines stay with them; a deleted row leaves its +/// lines in place, in front of the next surviving row (or of the trailing +/// lines); rows appended by INSERT go after the last row and before the +/// trailing lines. #[derive(Debug, Clone, Default, PartialEq, Eq)] pub struct CommentLines { /// Lines before the first record (the header, or the first row). @@ -809,6 +525,7 @@ impl Default for FileLayout { fn default() -> Self { FileLayout { has_header: true, + bom: false, crlf: false, comments: false, } @@ -826,6 +543,32 @@ pub enum DataType { Text, /// Boolean Boolean, + /// NUMERIC affinity (a declared type matching no other rule) + Numeric, + /// BLOB affinity (a declared type containing `BLOB`) + Blob, +} + +impl DataType { + /// The affinity of a declared column type, by SQLite's rules (in order): + /// `INT` anywhere gives INTEGER; `CHAR`, `CLOB` or `TEXT` gives TEXT; + /// `BLOB` or no type gives BLOB; `REAL`, `FLOA` or `DOUB` gives REAL; + /// anything else gives NUMERIC. Case-insensitive. + pub fn from_declared(decl: &str) -> DataType { + let t = decl.to_ascii_uppercase(); + let has = |s: &str| t.contains(s); + if has("INT") { + DataType::Integer + } else if has("CHAR") || has("CLOB") || has("TEXT") { + DataType::Text + } else if has("BLOB") || t.trim().is_empty() { + DataType::Blob + } else if has("REAL") || has("FLOA") || has("DOUB") { + DataType::Float + } else { + DataType::Numeric + } + } } impl fmt::Display for DataType { @@ -835,6 +578,8 @@ impl fmt::Display for DataType { DataType::Float => write!(f, "REAL"), DataType::Text => write!(f, "TEXT"), DataType::Boolean => write!(f, "BOOLEAN"), + DataType::Numeric => write!(f, "NUMERIC"), + DataType::Blob => write!(f, "BLOB"), } } } @@ -844,13 +589,27 @@ impl fmt::Display for DataType { pub struct Column { /// Name of the column pub name: String, - /// Data type of the column + /// Data type of the column (for a declared column, its affinity) pub data_type: DataType, + /// The type name as declared by CREATE TABLE / ALTER TABLE ADD COLUMN; + /// `None` for a loaded or result column + pub decl_type: Option, +} + +impl Column { + /// The type name `.schema` shows: the declared name when there is one, + /// else the data type's name. + pub fn type_name(&self) -> String { + match &self.decl_type { + Some(decl) => decl.clone(), + None => self.data_type.to_string(), + } + } } impl fmt::Display for Column { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - write!(f, "{} ({})", self.name, self.data_type) + write!(f, "{} ({})", self.name, self.type_name()) } } @@ -861,6 +620,8 @@ pub struct ColumnDefinition { pub name: String, /// Data type of the column pub data_type: DataType, + /// The declared type name, if the column was declared + pub decl_type: Option, } impl Table { @@ -868,26 +629,19 @@ impl Table { /// /// This constructor defaults all columns to Text type pub fn new(name: &str, column_names: Vec, file_path: Option) -> Self { - // Create column_map from column names - let column_map = column_names - .iter() - .enumerate() - .map(|(i, name)| (name.clone(), i)) - .collect(); - // Create full column metadata objects let cols = column_names .iter() .map(|name| Column { name: name.clone(), data_type: DataType::Text, + decl_type: None, }) .collect(); Table { name: name.to_string(), cols, - column_map, rows: Vec::new(), file_path, modified: false, @@ -895,6 +649,8 @@ impl Table { format: FileLayout::default(), comments: CommentLines::default(), placeholder: false, + raw: None, + name_index: OnceLock::new(), } } @@ -917,29 +673,19 @@ impl Table { file_path: Option, delimiter: Option, ) -> Self { - // Extract column names from the schema - let columns: Vec = schema.iter().map(|col_def| col_def.name.clone()).collect(); - // Create full column metadata objects let cols = schema .iter() .map(|col_def| Column { name: col_def.name.clone(), data_type: col_def.data_type, + decl_type: col_def.decl_type.clone(), }) .collect(); - // Create column_map from column names - let column_map = columns - .iter() - .enumerate() - .map(|(i, name)| (name.clone(), i)) - .collect(); - Table { name: name.to_string(), cols, - column_map, rows: Vec::new(), file_path, modified: true, // Tables created with schema are considered modified @@ -947,17 +693,32 @@ impl Table { format: FileLayout::default(), comments: CommentLines::default(), placeholder: false, + raw: None, + name_index: OnceLock::new(), } } + /// Give the columns declared names and types (CREATE TABLE ... LOCATION + /// over an existing file). `schema` has one entry per column. + pub fn set_declared_columns(&mut self, schema: Vec) { + debug_assert_eq!(schema.len(), self.cols.len()); + self.name_index = OnceLock::new(); + self.cols = schema + .into_iter() + .map(|c| Column { + name: c.name, + data_type: c.data_type, + decl_type: c.decl_type, + }) + .collect(); + } + /// Get the columns of the table /// /// Returns a vector containing all column names in the table. /// The column names maintain their original order as specified when /// the table was created or loaded from a file. pub fn columns(&self) -> Vec { - // Get column names from Column objects - // This prepares for future removal of the columns field self.cols.iter().map(|col| col.name.clone()).collect() } @@ -974,7 +735,7 @@ impl Table { /// validation when adding rows or performing operations that need to /// check column bounds. pub fn column_count(&self) -> usize { - self.columns().len() + self.cols.len() } /// Get the rows of the table @@ -1035,6 +796,9 @@ impl Table { } self.rows.push(row); + if let Some(raw) = &mut self.raw { + raw.rows.push(RawSpan::NONE); + } self.modified = true; Ok(()) } @@ -1104,32 +868,63 @@ impl Table { self.comments.flush_from(self.rows.len()); } - /// Get the index of a column by name - /// - /// Looks up a column by name and returns its index in the table. - /// This is essential for implementing SQL operations that reference - /// columns by name rather than position. + /// The index of the column called `name`. /// - /// # Arguments - /// * `name` - The name of the column to look up - /// - /// # Returns - /// * `Some(usize)` with the column index if found - /// * `None` if no column with that name exists + /// Column names are case-insensitive (ASCII), as in SQLite; this is the + /// one lookup every statement uses, so a name resolves the same way in a + /// SELECT, an INSERT column list and an UPDATE SET. A loaded or created + /// table cannot have two columns that match the same name. pub fn column_index(&self, name: &str) -> Option { - self.column_map.get(name).copied() + // A short list is scanned; a wide one is looked up in a map. + if self.cols.len() <= 16 { + return self + .cols + .iter() + .position(|c| c.name.eq_ignore_ascii_case(name)); + } + let index = self.name_index.get_or_init(|| { + let mut map = HashMap::with_capacity(self.cols.len()); + for (i, c) in self.cols.iter().enumerate() { + map.entry(c.name.to_ascii_lowercase()).or_insert(i); + } + map + }); + index.get(&name.to_ascii_lowercase()).copied() + } + + /// Rename the columns in order, as a CTE's column list does. `names` + /// has one entry per column. + pub fn rename_columns(&mut self, names: &[String]) { + debug_assert_eq!(names.len(), self.cols.len()); + self.name_index = OnceLock::new(); + for (col, name) in self.cols.iter_mut().zip(names) { + col.name = name.clone(); + } + self.raw = None; + } + + /// Make the column names unique as SQLite names the columns of a + /// derived table or CREATE TABLE ... AS (see [`unique_column_names`]). + pub fn dedup_column_names(&mut self) { + let mut names: Vec = self.cols.iter().map(|c| c.name.clone()).collect(); + unique_column_names(&mut names); + self.name_index = OnceLock::new(); + for (col, name) in self.cols.iter_mut().zip(names) { + col.name = name; + } + self.raw = None; } /// Print the table to stdout as a delimited file /// - /// Header line, then one record per row, using this table's delimiter - /// with RFC 4180 quoting where a field needs it. NULL prints as an empty - /// field. Returns the raw io::Error so the caller can recognise a closed - /// pipe (`sqawk ... | head`) and stop quietly. - pub fn print_to_stdout(&self) -> std::io::Result<()> { + /// Header line (when `header`), then one record per row, using this + /// table's delimiter with RFC 4180 quoting where a field needs it. NULL + /// prints as an empty field. Returns the raw io::Error so the caller can + /// recognise a closed pipe (`sqawk ... | head`) and stop quietly. + pub fn print_to_stdout(&self, header: bool) -> std::io::Result<()> { let stdout = std::io::stdout(); let mut out = std::io::BufWriter::new(stdout.lock()); - self.write_delimited(&mut out, true, false, false, None)?; + self.write_delimited(&mut out, header, false, false, None)?; std::io::Write::flush(&mut out) } @@ -1147,24 +942,70 @@ impl Table { crlf: bool, comments: bool, lines: Option<&CommentLines>, + ) -> std::io::Result<()> { + self.write_records(out, header, crlf, comments, lines, None) + } + + /// `write_delimited`, writing each record that has source bytes in + /// `raw` as those bytes. + fn write_records( + &self, + out: &mut W, + header: bool, + crlf: bool, + comments: bool, + lines: Option<&CommentLines>, + raw: Option<&RawRows>, ) -> std::io::Result<()> { let delim = delimiter_byte(&self.delimiter); + let terminator: &'static [u8] = if crlf { b"\r\n" } else { b"\n" }; let w = RecordWriter { delimiter: delim, - terminator: if crlf { b"\r\n" } else { b"\n" }, + terminator, comments, }; let empty = CommentLines::default(); let lines = lines.unwrap_or(&empty); - out.write_all(&lines.head)?; + let source = |span: RawSpan| raw.and_then(|r| span.get(&r.buf)); + // The last record of a file may have no line ending. Its bytes are + // written as they were, so anything written after it must first end + // its line. + let mut open_line = false; + let put = |out: &mut W, bytes: &[u8], open: &mut bool| -> std::io::Result<()> { + if bytes.is_empty() { + return Ok(()); + } + if *open { + out.write_all(terminator)?; + } + out.write_all(bytes)?; + *open = !matches!(bytes.last(), Some(b'\n' | b'\r')); + Ok(()) + }; + put(out, &lines.head, &mut open_line)?; if header { - let names = self.columns(); - w.write(out, names.iter().map(String::as_str))?; + match source(raw.map_or(RawSpan::NONE, |r| r.header)) { + Some(bytes) => put(out, bytes, &mut open_line)?, + None => { + let names = self.columns(); + let mut rec = Vec::new(); + w.write(&mut rec, names.iter().map(String::as_str))?; + put(out, &rec, &mut open_line)?; + } + } } let mut fields: Vec> = Vec::with_capacity(self.cols.len()); for (i, row) in self.rows.iter().enumerate() { if let Some(before) = lines.before.get(&i) { - out.write_all(before)?; + put(out, before, &mut open_line)?; + } + if let Some(bytes) = source(raw.map_or(RawSpan::NONE, |r| r.rows[i])) { + put(out, bytes, &mut open_line)?; + continue; + } + if open_line { + out.write_all(terminator)?; + open_line = false; } fields.clear(); fields.extend(row.iter().map(Value::field_str)); @@ -1173,24 +1014,36 @@ impl Table { // A comment line anchored past the last row (none should be) is // still written rather than lost. for (_, before) in lines.before.range(self.rows.len()..) { - out.write_all(before)?; + put(out, before, &mut open_line)?; } - out.write_all(&lines.tail) + put(out, &lines.tail, &mut open_line) } - /// Write the table back to its file format: `write_delimited` with the - /// recorded layout and comment lines. + /// Write the table back to its file format: the recorded layout (byte + /// order mark, header, line endings) and comment lines, with every row + /// that still has its source bytes written as those bytes. pub fn write_source_layout(&self, out: &mut W) -> std::io::Result<()> { let layout = self.file_layout(); - self.write_delimited( + if layout.bom { + out.write_all(b"\xef\xbb\xbf")?; + } + self.write_records( out, layout.has_header, layout.crlf, layout.comments, Some(&self.comments), + self.raw.as_deref(), ) } + /// Record the source bytes of the header and rows (see [`RawRows`]). + /// `raw.rows` must have one span per row. + pub fn set_raw(&mut self, raw: RawRows) { + debug_assert_eq!(raw.rows.len(), self.rows.len()); + self.raw = Some(Box::new(raw)); + } + /// Replace all rows with a new set /// /// This method is useful for operations like DELETE that need to replace @@ -1200,6 +1053,7 @@ impl Table { /// * `new_rows` - The new set of rows to replace the existing ones pub fn replace_rows(&mut self, new_rows: Vec) { self.rows = new_rows; + self.raw = None; self.comments.flush_from(self.rows.len()); self.modified = true; } @@ -1210,6 +1064,13 @@ impl Table { /// move to the next surviving row, or to the tail. pub fn retain_rows(&mut self, keep: impl Fn(usize) -> bool) { let old = std::mem::take(&mut self.rows); + if let Some(raw) = &mut self.raw { + let mut i = 0; + raw.rows.retain(|_| { + i += 1; + keep(i - 1) + }); + } let mut before = std::mem::take(&mut self.comments.before); let mut pending: Vec = Vec::new(); for (i, row) in old.into_iter().enumerate() { @@ -1245,6 +1106,16 @@ impl Table { row_index, self.name ))); } + if let Some(raw) = &mut self.raw { + let same = rows[row_index].len() == row.len() + && rows[row_index] + .iter() + .zip(&row) + .all(|(a, b)| identical(a, b)); + if !same { + raw.rows[row_index] = RawSpan::NONE; + } + } rows[row_index] = row; self.modified = true; Ok(()) @@ -1260,29 +1131,33 @@ impl Table { _ => DataType::Text, }; let column = Column { - name: name.clone(), + name, data_type, + decl_type: None, }; + self.name_index = OnceLock::new(); self.cols.push(column); - let new_index = self.cols.len() - 1; - self.column_map.insert(name, new_index); + self.raw = None; self.modified = true; } - /// Add a new column with a default value for all existing rows + /// Add a new column, declared as `decl_type`, with a default value for + /// all existing rows pub fn add_column_with_default( &mut self, name: String, - data_type: DataType, + decl_type: String, default_value: Value, ) -> crate::error::SqawkResult<()> { let column = Column { - name: name.clone(), - data_type, + name, + data_type: DataType::from_declared(&decl_type), + decl_type: Some(decl_type), }; + self.name_index = OnceLock::new(); self.cols.push(column); - let new_index = self.cols.len() - 1; - self.column_map.insert(name, new_index); + // Every line of the file gains a field. + self.raw = None; // Add default value to all existing rows for row in &mut self.rows { @@ -1296,28 +1171,14 @@ impl Table { /// Clear all rows from the table (TRUNCATE TABLE) pub fn clear_rows(&mut self) -> crate::error::SqawkResult<()> { self.rows.clear(); + // The header line is still written back as it was read. + if let Some(raw) = &mut self.raw { + raw.rows.clear(); + } self.comments.flush_from(0); self.modified = true; Ok(()) } - - /// Convert all rows to string representation - /// - /// This method converts all table rows to a vector of string vectors, - /// useful for displaying in the REPL interface. - /// - /// # Returns - /// * Vec> - All rows converted to strings - pub fn rows_as_strings(&self) -> Vec> { - self.rows() - .iter() - .map(|row| { - row.iter() - .map(|value| value.field_str().into_owned()) - .collect() - }) - .collect() - } } /// The delimiter byte for a table's delimiter string. @@ -1397,3 +1258,21 @@ impl RecordWriter { out.write_all(self.terminator) } } + +#[cfg(test)] +mod tests { + use super::unique_column_names; + + #[test] + fn unique_column_names_like_sqlite() { + let mut names: Vec = ["Id", "id", "id+1", "ID", "id+1", "a:1", "a:1", "x:"] + .iter() + .map(|s| s.to_string()) + .collect(); + unique_column_names(&mut names); + assert_eq!( + names, + ["Id", "id:1", "id+1", "ID:2", "id+1:1", "a:1", "a:2", "x:"] + ); + } +} diff --git a/src/vm/agg.rs b/src/vm/agg.rs index e58f46a..129c4b1 100644 --- a/src/vm/agg.rs +++ b/src/vm/agg.rs @@ -3,7 +3,7 @@ use std::collections::HashSet; -use super::arith::text_of; +use super::arith::{self, text_of}; use super::bytecode::{ Register, AGG_AVG, AGG_COUNT, AGG_DISTINCT, AGG_GROUP_CONCAT, AGG_MAX, AGG_MIN, AGG_SUM, AGG_TOTAL, AGG_TYPE_MASK, @@ -12,22 +12,81 @@ use super::engine::{compare_values, row_key, RowKey}; use crate::error::{SqawkError, SqawkResult}; use crate::table::Value; -/// A running sum: exact while every addend is an integer and the total fits. +/// A running sum: exact while every addend is an integer and the total fits, +/// else a Kahan-Babuska-Neumaier compensated sum (as SQLite's SUM, TOTAL +/// and AVG), so `0.1 + 0.2 + 0.3` is 0.6. #[derive(Debug, Clone, Copy)] enum Sum { Int(i64), - Real(f64), + /// Running sum and the accumulated rounding error. + Real { + sum: f64, + err: f64, + }, } +/// Integers at or beyond 2^52 in magnitude are split before being added in +/// floating point, so their low bits survive in the error term (SQLite's +/// `kahanBabuskaNeumaierStepInt64`). +const SPLIT_INT: i64 = 4_503_599_627_370_496; + impl Sum { + /// The compensated sum of an integer total, ready for real addends. + fn from_int(i: i64) -> Sum { + if i <= -SPLIT_INT || i >= SPLIT_INT { + let small = i % 16384; + Sum::Real { + sum: (i - small) as f64, + err: small as f64, + } + } else { + Sum::Real { + sum: i as f64, + err: 0.0, + } + } + } + + /// Add a real (Neumaier's step). Only called on `Sum::Real`. + fn add_real(&mut self, r: f64) { + if let Sum::Real { sum, err } = self { + let t = *sum + r; + if sum.abs() > r.abs() { + *err += (*sum - t) + r; + } else { + *err += (r - t) + *sum; + } + *sum = t; + } + } + + /// Add an integer to a compensated sum. + fn add_int(&mut self, i: i64) { + if i <= -SPLIT_INT || i >= SPLIT_INT { + let small = i % 16384; + self.add_real((i - small) as f64); + self.add_real(small as f64); + } else { + self.add_real(i as f64); + } + } + fn as_f64(self) -> f64 { match self { Sum::Int(i) => i as f64, - Sum::Real(f) => f, + // An infinite or NaN error term is dropped, as SQLite drops it. + Sum::Real { sum, err } if err.is_finite() => sum + err, + Sum::Real { sum, .. } => sum, } } } +/// SUM / AVG operand. +enum Addend { + Int(i64), + Real(f64), +} + /// An aggregate's running state. #[derive(Debug, Clone)] pub(super) struct AggAccumulator { @@ -37,6 +96,10 @@ pub(super) struct AggAccumulator { count: i64, /// SUM / AVG / TOTAL. sum: Option, + /// SUM: the integer sum overflowed and no real has been added since + /// (SQLite reports "integer overflow" only if that is still so at the + /// end). + overflow: bool, /// MIN / MAX. best: Option, /// GROUP_CONCAT. @@ -50,10 +113,10 @@ pub(super) struct AggAccumulator { /// SUM / AVG operand: numbers as they are, numeric text as its number. /// `None` (skipped) for NULL and for text with no numeric reading. -fn sum_operand(v: &Value) -> Option { +fn sum_operand(v: &Value) -> Option { match v.as_numeric()? { - Value::Integer(i) => Some(Sum::Int(i)), - Value::Float(f) => Some(Sum::Real(f)), + Value::Integer(i) => Some(Addend::Int(i)), + Value::Float(f) => Some(Addend::Real(f)), _ => None, } } @@ -66,6 +129,7 @@ impl AggAccumulator { func_type: func_type & AGG_TYPE_MASK, count: 0, sum: None, + overflow: false, best: None, text: None, seen: (func_type & AGG_DISTINCT != 0).then(Default::default), @@ -93,19 +157,27 @@ impl AggAccumulator { return Ok(()); }; self.count += 1; - self.sum = Some(match (self.sum, v) { - (None, v) => v, - (Some(Sum::Int(a)), Sum::Int(b)) => match a.checked_add(b) { - Some(s) => Sum::Int(s), - // SQLite: SUM of integers that overflows is an - // error; AVG and TOTAL carry on in floating point. - None if self.func_type == AGG_SUM => { - return Err(SqawkError::InvalidSqlQuery("integer overflow".into())) + // SQLite's sumStep. + let sum = self.sum.get_or_insert(Sum::Int(0)); + match (*sum, v) { + (Sum::Int(a), Addend::Int(b)) => match a.checked_add(b) { + Some(s) => *sum = Sum::Int(s), + None => { + self.overflow = true; + *sum = Sum::from_int(a); + sum.add_int(b); } - None => Sum::Real(a as f64 + b as f64), }, - (Some(a), b) => Sum::Real(a.as_f64() + b.as_f64()), - }); + (Sum::Int(a), Addend::Real(r)) => { + *sum = Sum::from_int(a); + sum.add_real(r); + } + (Sum::Real { .. }, Addend::Int(b)) => sum.add_int(b), + (Sum::Real { .. }, Addend::Real(r)) => { + self.overflow = false; + sum.add_real(r); + } + } } AGG_MIN | AGG_MAX => { if let Some(val) = value.filter(|v| !matches!(v, Value::Null)) { @@ -151,10 +223,57 @@ impl AggAccumulator { Ok(()) } - /// The aggregate's value. Of no rows, COUNT is 0, TOTAL 0.0 and the - /// rest NULL. - pub(super) fn finalize(&self) -> Value { + /// Remove one value accumulated earlier -- the oldest, as a window + /// frame's start moves -- for COUNT, SUM, AVG and TOTAL (SQLite's + /// `countInverse` / `sumInverse`). The state is not rewound: once a real + /// has been added SUM stays a real, and reals are subtracted in the + /// compensated sum, so the value is the one SQLite computes. + pub(super) fn inverse(&mut self, value: Option) { match self.func_type { + AGG_COUNT => { + if !matches!(value, Some(Value::Null)) { + self.count -= 1; + } + } + AGG_SUM | AGG_AVG | AGG_TOTAL => { + let Some(v) = value.as_ref().and_then(sum_operand) else { + return; + }; + self.count -= 1; + let Some(sum) = self.sum.as_mut() else { + return; + }; + match (*sum, v) { + (Sum::Int(a), Addend::Int(b)) => *sum = Sum::Int(a.wrapping_sub(b)), + (Sum::Int(a), Addend::Real(r)) => { + *sum = Sum::Int(a.wrapping_sub(arith::real_to_int(r))) + } + (Sum::Real { .. }, Addend::Int(b)) => { + if b == i64::MIN { + sum.add_int(i64::MAX); + sum.add_int(1); + } else { + sum.add_int(-b); + } + } + (Sum::Real { .. }, Addend::Real(r)) => sum.add_real(-r), + } + } + // MIN / MAX and GROUP_CONCAT frames are handled by the window + // code itself. + _ => {} + } + } + + /// The aggregate's value. Of no rows, COUNT is 0, TOTAL 0.0 and the + /// rest NULL. A SUM of integers that overflowed (with no real added + /// after the overflow) is an "integer overflow" error, as in SQLite; + /// AVG and TOTAL carry on in floating point. + pub(super) fn finalize(&self) -> SqawkResult { + if self.func_type == AGG_SUM && self.overflow && self.count > 0 { + return Err(arith::overflow()); + } + Ok(match self.func_type { AGG_COUNT => Value::Integer(self.count), AGG_AVG => match self.sum { Some(s) if self.count > 0 => Value::Float(s.as_f64() / self.count as f64), @@ -162,8 +281,9 @@ impl AggAccumulator { }, AGG_TOTAL => Value::Float(self.sum.map_or(0.0, Sum::as_f64)), AGG_SUM => match self.sum { + _ if self.count == 0 => Value::Null, Some(Sum::Int(i)) => Value::Integer(i), - Some(Sum::Real(f)) => Value::Float(f), + Some(s) => Value::Float(s.as_f64()), None => Value::Null, }, AGG_GROUP_CONCAT => match &self.text { @@ -171,6 +291,6 @@ impl AggAccumulator { None => Value::Null, }, _ => self.best.clone().unwrap_or(Value::Null), - } + }) } } diff --git a/src/vm/arith.rs b/src/vm/arith.rs index 971649e..caa16fb 100644 --- a/src/vm/arith.rs +++ b/src/vm/arith.rs @@ -100,7 +100,7 @@ pub(crate) fn text_to_num(s: &str) -> Num { return Num::Int(i); } } - Num::Real(text.parse::().unwrap_or(0.0)) + Num::Real(atof(text)) } /// Text as an integer, the way `CAST(text AS INTEGER)` reads it: an optional @@ -162,6 +162,62 @@ pub(crate) fn to_num(r: &Register) -> Option { } } +/// A register as a 64-bit integer, as SQLite's `sqlite3VdbeIntValue` reads +/// it for the bitwise operators: text by its integer prefix (`'12abc'` is +/// 12, `'1e3'` is 1), reals truncated toward zero (saturating), NULL 0. +pub(crate) fn int_value(r: &Register) -> i64 { + match r { + Register::Integer(i) => *i, + Register::Boolean(b) => *b as i64, + Register::Float(f) => real_to_int(*f), + Register::String(s) => text_to_int(s), + Register::Null => 0, + } +} + +/// A bitwise operator. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) enum BitOp { + And, + Or, + ShiftLeft, + ShiftRight, +} + +/// `a op b` for `&`, `|`, `<<`, `>>` with SQLite semantics: NULL if either +/// side is NULL, else both sides as integers ([`int_value`]). A negative +/// shift count shifts the other way; shifting by 64 or more gives 0, or -1 +/// for `>>` of a negative number; `>>` extends the sign. +pub(crate) fn bitwise(op: BitOp, a: &Register, b: &Register) -> Register { + if matches!(a, Register::Null) || matches!(b, Register::Null) { + return Register::Null; + } + let ia = int_value(a); + let mut ib = int_value(b); + Register::Integer(match op { + BitOp::And => ia & ib, + BitOp::Or => ia | ib, + BitOp::ShiftLeft | BitOp::ShiftRight => { + let mut left = op == BitOp::ShiftLeft; + if ib < 0 { + left = !left; + ib = if ib > -64 { -ib } else { 64 }; + } + if ib >= 64 { + if ia >= 0 || left { + 0 + } else { + -1 + } + } else if left { + ((ia as u64) << ib) as i64 + } else { + ia >> ib + } + } + }) +} + /// The truth value of a register in a condition, as in SQLite: a number is /// true when non-zero, and text is read as a number first, so `'x'` is false /// and `'1x'` true. NULL is not true. @@ -216,6 +272,19 @@ pub(crate) fn arith(op: ArithOp, a: &Register, b: &Register) -> Register { None => real_arith(op, x as f64, y as f64), } } + // With a real operand SQLite still takes an integer remainder: of + // each operand read as an integer (`sqlite3VdbeIntValue` -- + // integers exactly, reals truncated and clamped to the i64 range, + // text by its integer prefix), returned as a real. 5.5 % 2 is 1.0, + // 9007199254740993 % 2.0 is 1.0. + _ if op == ArithOp::Rem => { + let (ix, iy) = (int_value(a), int_value(b)); + if iy == 0 { + return Register::Null; + } + let iy = if iy == -1 { 1 } else { iy }; + Register::Float((ix % iy) as f64) + } (x, y) => real_arith(op, x.as_f64(), y.as_f64()), } } @@ -232,7 +301,8 @@ fn real_arith(op: ArithOp, x: f64, y: f64) -> Register { x / y } // SQLite takes the remainder of the operands truncated to integers, - // but returns it as a real: 5.5 % 2 is 1.0. + // but returns it as a real (reached only from `arith`'s integer + // path, which never overflows for `%`). ArithOp::Rem => { let (ix, iy) = (real_to_int(x), real_to_int(y)); if iy == 0 { @@ -282,12 +352,371 @@ pub(crate) fn cast(r: Register, target: &str) -> Register { } } +// The error of each power of ten as a double (the exact power minus the +// double), the low halves of SQLite's double-double constants, spelled as +// SQLite spells them. +#[allow(clippy::excessive_precision)] +const E100_LO: f64 = -1.5902891109759918046e+83; +#[allow(clippy::excessive_precision)] +const EM100_LO: f64 = -1.99918998026028836196e-117; +#[allow(clippy::excessive_precision)] +const EM10_LO: f64 = -3.6432197315497741579e-27; +#[allow(clippy::excessive_precision)] +const EM1_LO: f64 = -5.5511151231257827021e-18; + +/// Double-double multiplication `x *= (y + yy)` (T. J. Dekker, "A +/// Floating-Point Technique for Extending the Available Precision", 1971), +/// SQLite's `dekkerMul2`. Every operation rounds to binary64 (Rust never +/// fuses or widens f64 arithmetic), as the algorithm requires. +fn dekker_mul2(x: &mut [f64; 2], y: f64, yy: f64) { + const HIGH: u64 = 0xffff_ffff_fc00_0000; + let hx = f64::from_bits(x[0].to_bits() & HIGH); + let tx = x[0] - hx; + let hy = f64::from_bits(y.to_bits() & HIGH); + let ty = y - hy; + let p = hx * hy; + let q = hx * ty + tx * hy; + let c = p + q; + let cc = p - c + q + tx * ty; + let cc = x[0] * yy + x[1] * y + cc; + x[0] = c + cc; + x[1] = c - x[0]; + x[1] += cc; +} + +/// The numeric prefix of `s` (leading whitespace, sign, digits, point, +/// exponent) as a real, computed as SQLite's `sqlite3AtoF` (3.50; the +/// double-double path of 3.45 builds without long double) computes it: +/// up to 19 significant digits as an integer, then scaled by powers of +/// ten in double-double arithmetic. The result is the same on every +/// platform and is what SQLite reads for the same text, which is not +/// always the correctly rounded double (`1e-300`'s neighbour can come +/// back for 17-digit input near the ends of the range). Text without a +/// number reads as 0.0. +// The bounds are spelled as SQLite spells them. +#[allow(clippy::excessive_precision)] +pub(crate) fn atof(s: &str) -> f64 { + const LIMIT: u64 = (u64::MAX - 9) / 10; + let b = s.as_bytes(); + let mut i = 0; + while i < b.len() && is_space(b[i]) { + i += 1; + } + let mut neg = false; + if i < b.len() && (b[i] == b'-' || b[i] == b'+') { + neg = b[i] == b'-'; + i += 1; + } + let mut sig: u64 = 0; + let mut d: i64 = 0; + while i < b.len() && b[i].is_ascii_digit() { + sig = sig * 10 + u64::from(b[i] - b'0'); + i += 1; + if sig >= LIMIT { + // Further integer digits only shift the point. + while i < b.len() && b[i].is_ascii_digit() { + i += 1; + d += 1; + } + } + } + if i < b.len() && b[i] == b'.' { + i += 1; + while i < b.len() && b[i].is_ascii_digit() { + if sig < LIMIT { + sig = sig * 10 + u64::from(b[i] - b'0'); + d -= 1; + } + i += 1; + } + } + let mut e: i64 = 0; + if i < b.len() && (b[i] == b'e' || b[i] == b'E') { + i += 1; + let mut eneg = false; + if i < b.len() && (b[i] == b'-' || b[i] == b'+') { + eneg = b[i] == b'-'; + i += 1; + } + while i < b.len() && b[i].is_ascii_digit() { + e = if e < 10000 { + e * 10 + i64::from(b[i] - b'0') + } else { + 10000 + }; + i += 1; + } + if eneg { + e = -e; + } + } + let sign = |f: f64| if neg { -f } else { f }; + if sig == 0 { + return sign(0.0); + } + e += d; + while e > 0 && sig < (u64::MAX - 0x7ff) / 10 { + sig *= 10; + e -= 1; + } + while e < 0 && sig.is_multiple_of(10) { + sig /= 10; + e += 1; + } + let mut rr = [sig as f64, 0.0]; + // The largest double that converts to u64 safely. + if rr[0] <= 18_446_744_073_709_549_568.0 { + let s2 = rr[0] as u64; + rr[1] = if sig >= s2 { + (sig - s2) as f64 + } else { + -((s2 - sig) as f64) + }; + } + if e > 0 { + while e >= 100 { + e -= 100; + dekker_mul2(&mut rr, 1.0e+100, E100_LO); + } + while e >= 10 { + e -= 10; + dekker_mul2(&mut rr, 1.0e+10, 0.0); + } + while e >= 1 { + e -= 1; + dekker_mul2(&mut rr, 1.0e+01, 0.0); + } + } else { + while e <= -100 { + e += 100; + dekker_mul2(&mut rr, 1.0e-100, EM100_LO); + } + while e <= -10 { + e += 10; + dekker_mul2(&mut rr, 1.0e-10, EM10_LO); + } + while e <= -1 { + e += 1; + dekker_mul2(&mut rr, 1.0e-01, EM1_LO); + } + } + let mut r = rr[0] + rr[1]; + if r.is_nan() { + r = f64::INFINITY; + } + sign(r) +} + +/// SQLite's `sqlite3Isspace`: ASCII space, \t \n \v \f \r. +fn is_space(c: u8) -> bool { + matches!(c, b' ' | b'\t' | b'\n' | 0x0b | 0x0c | b'\r') +} + +/// A real decoded to decimal digits: SQLite's `FpDecode`. +pub(crate) struct FpDecode { + /// `b'+'` or `b'-'`. + pub(crate) sign: u8, + /// 0: finite; 1: infinity; 2: NaN. + pub(crate) special: u8, + buf: [u8; 24], + start: usize, + /// Number of significant digits (trailing zeros removed). + n: usize, + /// Position of the decimal point relative to the first digit. + pub(crate) idp: i64, +} + +impl FpDecode { + /// The significant digits, ASCII, without trailing zeros. + pub(crate) fn digits(&self) -> &[u8] { + &self.buf[self.start..self.start + self.n] + } +} + +/// SQLite's `sqlite3FpDecode` (3.50, which always uses the portable +/// double-double path that 3.45 builds without long double use): the +/// magnitude is scaled by powers of ten in double-double arithmetic until +/// it lies near 1e18, read as an integer of 18 or 19 digits, and those +/// digits are rounded half up to `iround` significant digits (when +/// `iround` is negative, to `-iround` digits after the point; 0 means no +/// rounding) and to at most `mxround` digits. The result is the same on +/// every platform. +// The bounds are spelled as SQLite spells them. +#[allow(clippy::excessive_precision)] +pub(crate) fn fp_decode(r: f64, iround: i64, mxround: i64) -> FpDecode { + let mut p = FpDecode { + sign: b'+', + special: 0, + buf: [b'0'; 24], + start: 0, + n: 1, + idp: 1, + }; + let mut r = r; + if r < 0.0 { + p.sign = b'-'; + r = -r; + } else if r == 0.0 { + return p; + } + if !r.is_finite() { + p.special = if r.is_nan() { 2 } else { 1 }; + p.n = 0; + p.idp = 0; + return p; + } + // Scale into [9.2e17, 9.2e18]; the low-order constants are the error + // of each power of ten as a double, as SQLite has them. + let mut exp: i64 = 0; + let mut rr = [r, 0.0]; + if rr[0] > 9.223_372_036_854_774_784e18 { + while rr[0] > 9.223_372_036_854_774_784e118 { + exp += 100; + dekker_mul2(&mut rr, 1.0e-100, EM100_LO); + } + while rr[0] > 9.223_372_036_854_774_784e28 { + exp += 10; + dekker_mul2(&mut rr, 1.0e-10, EM10_LO); + } + while rr[0] > 9.223_372_036_854_774_784e18 { + exp += 1; + dekker_mul2(&mut rr, 1.0e-01, EM1_LO); + } + } else { + while rr[0] < 9.223_372_036_854_774_784e-83 { + exp -= 100; + dekker_mul2(&mut rr, 1.0e+100, E100_LO); + } + while rr[0] < 9.223_372_036_854_774_784e7 { + exp -= 10; + dekker_mul2(&mut rr, 1.0e+10, 0.0); + } + while rr[0] < 9.223_372_036_854_774_78e17 { + exp -= 1; + dekker_mul2(&mut rr, 1.0e+01, 0.0); + } + } + let mut v: u64 = if rr[1] < 0.0 { + (rr[0] as u64).wrapping_sub((-rr[1]) as u64) + } else { + (rr[0] as u64).wrapping_add(rr[1] as u64) + }; + + // The digits of v, right-aligned in buf; buf[i] is the free slot + // before the first one. + let mut i = p.buf.len() - 1; + while v != 0 { + p.buf[i] = b'0' + (v % 10) as u8; + i -= 1; + v /= 10; + } + let mut n = (p.buf.len() - 1 - i) as i64; + p.idp = n + exp; + let mut iround = iround; + if iround < 0 { + iround = p.idp - iround; + if iround == 0 && p.buf[i + 1] >= b'5' { + iround = 1; + p.buf[i] = b'0'; + i -= 1; + n += 1; + p.idp += 1; + } + } + if iround > 0 && (iround < n || n > mxround) { + let z = i + 1; + if iround > mxround { + iround = mxround; + } + n = iround; + if p.buf[z + iround as usize] >= b'5' { + let mut j = z + iround as usize - 1; + loop { + p.buf[j] += 1; + if p.buf[j] <= b'9' { + break; + } + p.buf[j] = b'0'; + if j == z { + p.buf[i] = b'1'; + i -= 1; + n += 1; + p.idp += 1; + break; + } + j -= 1; + } + } + } + p.start = i + 1; + let mut n = n as usize; + while n > 1 && p.buf[p.start + n - 1] == b'0' { + n -= 1; + } + p.n = n; + p +} + +/// A REAL's text, exactly as SQLite's `CAST(x AS TEXT)` renders it +/// (`%!.15g`): 15 significant digits rounded half up from the +/// [`fp_decode`] expansion, an integral value keeps a `.0` (`2.0`, +/// `1.0e+20`), infinities are `Inf` / `-Inf`, and negative zero is `0.0`. +/// NaN, which SQLite turns into NULL, renders as the empty string. +pub(crate) fn real_text(f: f64) -> String { + if f.is_nan() { + return String::new(); + } + if f.is_infinite() { + return if f > 0.0 { "Inf" } else { "-Inf" }.into(); + } + if f == 0.0 { + return "0.0".into(); + } + let d = fp_decode(f, 15, 26); + let z = d.digits(); + let x = d.idp - 1; + let mut out = String::with_capacity(24); + if d.sign == b'-' { + out.push('-'); + } + let push = |out: &mut String, b: &[u8]| out.extend(b.iter().map(|&c| c as char)); + // At most 15 digits, all of which fit in the 14 places `%.15g` shows + // after the first, so no digit is cut here. + if !(-4..=14).contains(&x) { + push(&mut out, &z[..1]); + out.push('.'); + push(&mut out, if z.len() > 1 { &z[1..] } else { b"0" }); + out.push('e'); + out.push(if x < 0 { '-' } else { '+' }); + out.push_str(&format!("{:02}", x.unsigned_abs())); + } else if x < 0 { + out.push_str("0."); + out.extend(std::iter::repeat_n('0', (-x - 1) as usize)); + push(&mut out, z); + } else { + let int_len = x as usize + 1; + for k in 0..int_len { + out.push(z.get(k).map_or('0', |&c| c as char)); + } + out.push('.'); + push( + &mut out, + if z.len() > int_len { + &z[int_len..] + } else { + b"0" + }, + ); + } + out +} + /// A non-NULL register's text form, as functions and `||` see it. pub(crate) fn text_of(r: &Register) -> String { match r { Register::String(s) => s.clone(), Register::Integer(i) => i.to_string(), - Register::Float(f) => f.to_string(), + Register::Float(f) => real_text(*f), Register::Boolean(b) => (*b as i64).to_string(), Register::Null => String::new(), } @@ -357,6 +786,30 @@ mod tests { assert_eq!(n("99999999999999999999"), Num::Real(1e20)); } + /// Values checked against SQLite 3.50.2 and 3.45.0 built without long + /// double (tsq's oracle). + #[test] + fn double_double_conversions() { + assert_eq!(real_text(443.85 / 5867.0), "0.0756519515936595"); + assert_eq!(real_text(100.0 / 476.42), "0.209898828764536"); + assert_eq!(real_text(1e20), "1.0e+20"); + assert_eq!(real_text(-1e-5), "-1.0e-05"); + assert_eq!(real_text(5e-324), "4.94065645841247e-324"); + assert_eq!(real_text(f64::MAX), "1.79769313486232e+308"); + assert_eq!(real_text(123456789012345.0), "123456789012345.0"); + assert_eq!(real_text(0.0001), "0.0001"); + assert_eq!(atof("1e-300").to_bits(), 0x01a5_6e1f_c2f8_f358); + assert_eq!(atof("1.0e-300"), atof("1e-300")); + assert_eq!(atof(" -2.5e3x"), -2500.0); + assert_eq!(atof("0.1"), 0.1); + assert_eq!(atof("1e400"), f64::INFINITY); + assert_eq!(atof("1e-400"), 0.0); + assert_eq!(atof("."), 0.0); + let d = fp_decode(17.0 / 1.5, 21, 26); + assert_eq!(d.digits(), b"1133333333333333392"); + assert_eq!(d.idp, 2); + } + #[test] fn cast_int_prefix() { assert_eq!(text_to_int("3.9"), 3); diff --git a/src/vm/ast_compat.rs b/src/vm/ast_compat.rs index c5a6b1f..9cfd85c 100644 --- a/src/vm/ast_compat.rs +++ b/src/vm/ast_compat.rs @@ -194,6 +194,25 @@ pub(crate) fn sql_option_key_value(option: &SqlOption) -> Option<(String, String } } +/// The table an `ObjectName` names in a table position: `t` or `main.t`. +/// +/// Every table lives in SQLite's `main` schema (matched case-insensitively), +/// so that is the one schema qualifier accepted; any other schema, or a +/// catalog, names a table that does not exist. +pub(crate) fn main_table_name(name: &ObjectName) -> SqawkResult { + match name.0.as_slice() { + [_] => Ok(object_name_last(name)), + [schema, _] + if schema + .as_ident() + .is_some_and(|i| i.value.eq_ignore_ascii_case("main")) => + { + Ok(object_name_last(name)) + } + _ => Err(SqawkError::TableNotFound(name.to_string())), + } +} + /// The final segment of an `ObjectName` -- the bare table or function name. pub(crate) fn object_name_last(name: &ObjectName) -> String { match name.0.last() { diff --git a/src/vm/bytecode.rs b/src/vm/bytecode.rs index bb5d03f..f122dc6 100644 --- a/src/vm/bytecode.rs +++ b/src/vm/bytecode.rs @@ -86,12 +86,27 @@ impl ResultSchema { } } +/// Escape one field of a DDL instruction's P4 spec (CreateTable, +/// AlterTableAdd), whose fields are separated by `:` and `|`. Column names +/// may contain either -- an unaliased result column is named by its SQL +/// text, `a||b` -- and so may file paths. +pub fn spec_escape(field: &str) -> String { + field + .replace('%', "%25") + .replace(':', "%3A") + .replace('|', "%7C") +} + +/// Undo [`spec_escape`]. +pub fn spec_unescape(field: &str) -> String { + field + .replace("%7C", "|") + .replace("%3A", ":") + .replace("%25", "%") +} + /// Opcodes for VM instructions -/// -/// Note: Some opcodes are defined but not yet used by the compiler. -/// They are kept for planned features in the SQL expansion roadmap. #[derive(Debug, Clone, Copy, PartialEq)] -#[allow(dead_code)] pub enum OpCode { // Program flow control Init, // Initialize VM @@ -118,13 +133,6 @@ pub enum OpCode { ResultRow, // Return result row to client Copy, // Copy register P1 to register P2 (SCopy in SQLite) - // Transaction operations - Begin, // Begin a transaction - marks the start of a set of changes that can be committed or rolled back - Commit, // Commit a transaction - permanently applies all changes made since the Begin operation - Rollback, // Rollback a transaction - discards all changes made since the Begin operation - SavePoint, // Create a savepoint in the transaction - establishes a point to which a transaction can be partially rolled back - Release, // Release a savepoint - confirms changes up to the specified savepoint - // Comparison operations Lt, // Less than comparison (P1 < P2, result in P3) Le, // Less than or equal comparison (P1 <= P2, result in P3) @@ -174,7 +182,6 @@ pub enum OpCode { // Conditional jumps IfZ, // Jump to P2 if register P1 contains 0 IfPos, // Jump to P2 if register P1 is positive (> 0) - IfNeg, // Jump to P2 if register P1 is negative (< 0) // Utility opcodes Noop, // No operation @@ -234,14 +241,14 @@ pub enum OpCode { // no-op), Found/NotFound test membership, IdxDelete removes a row, and // Rewind walks the rows in sorted order. DISTINCT and the compound // operators (UNION, INTERSECT, EXCEPT) are built from these. Rows are - // equal when every column is: NULL equals NULL and 1 equals 1.0. - OpenEphemeral, // Open ephemeral cursor P1 with P2 columns. P4 = sort key spec; P3 = 1: keyed. + // equal when every column is: NULL equals NULL and 1 equals 1.0. With + // P3 = 2 columns are keyed as `=` compares instead (text '1' equals 1), + // for IN (SELECT ...). + OpenEphemeral, // Open ephemeral cursor P1 with P2 columns. P3 = 1/2: keyed. IdxInsert, // Insert into ephemeral cursor P1. P2 = start register, P3 = column count. IdxDelete, // Remove row r[P2..P2+P3] from keyed ephemeral cursor P1, if present. Found, // Jump to P2 if row r[P3..P3+P4] is in keyed ephemeral cursor P1. NotFound, // Jump to P2 if row r[P3..P3+P4] is NOT in keyed ephemeral cursor P1. - Sort, // Sort ephemeral cursor P1. Jump to P2 when empty, else continue. - Sequence, // P1 = ephemeral cursor, P2 = dest register. Generate next sequence number. // DDL operations CreateTable, // Create a table. P4 = "table_name:col1:type1,col2:type2,..." diff --git a/src/vm/compiler.rs b/src/vm/compiler.rs index 5073fca..f9888ca 100644 --- a/src/vm/compiler.rs +++ b/src/vm/compiler.rs @@ -5,17 +5,17 @@ //! It implements a visitor pattern to walk the AST generated by sqlparser. use super::ast_compat::{ - create_table_options, delete_from, func_args, func_is_distinct, insert_target, object_name_last, + create_table_options, delete_from, func_args, func_is_distinct, insert_target, main_table_name, + object_name_last, }; use sqlparser::ast::{ BinaryOperator, CaseWhen, DataType as SqlDataType, Expr, Function, FunctionArg, - FunctionArgExpr, FunctionArguments, Ident, ObjectName, SelectItem, + FunctionArgExpr, FunctionArguments, Ident, ObjectName, Select, SelectItem, SelectItemQualifiedWildcardKind, Statement, TrimWhereField, UnaryOperator, Value, ValueWithSpan, }; -use sqlparser::dialect::HiveDialect; -use sqlparser::parser::Parser; -use std::collections::HashMap; +use std::collections::{HashMap, HashSet}; +use std::sync::Arc; use super::bytecode::{Instruction, OpCode, Program, AGG_COUNT, AGG_DISTINCT}; use super::compiler_check::{check_create_table, check_delete, check_insert, check_update}; @@ -23,6 +23,7 @@ use super::compiler_from::{expr_children, FromPlan}; use super::compiler_subquery::Sub; use super::compiler_window::window_call_children; use super::func; +use super::parse::SqlSource; use crate::aggregate::AggregateFunction; use crate::capacity::DEFAULT_CURSOR_CAPACITY; use crate::database::Database; @@ -83,8 +84,97 @@ pub(crate) struct UsingCol { pub refs: Vec<(i64, usize)>, } +/// The USING / NATURAL merged columns of a FROM clause, indexed by name and +/// by joined column, so that a NATURAL JOIN of wide tables resolves each +/// column in constant time. +#[derive(Clone, Debug, Default)] +pub(crate) struct UsingCols { + cols: Vec, + /// ASCII-lowercased name -> index into `cols` (the first with the name). + by_name: HashMap, + /// Each merged (cursor, column) -> index into `cols`. + by_ref: HashMap<(i64, usize), usize>, +} + +impl UsingCols { + /// Add `(cursor, column)` to the merged column that holds `left`, or + /// start a new merged column `name` of `left` and `right`. + pub(crate) fn merge(&mut self, name: String, left: (i64, usize), right: (i64, usize)) { + let i = match self.by_ref.get(&left) { + Some(&i) => { + self.cols[i].refs.push(right); + i + } + None => { + let i = self.cols.len(); + self.by_name.entry(name.clone()).or_insert(i); + self.by_ref.insert(left, i); + self.cols.push(UsingCol { + name, + refs: vec![left, right], + }); + i + } + }; + self.by_ref.entry(right).or_insert(i); + } + + /// Index of the merged column named `name` (ASCII-lowercased). + pub(crate) fn position(&self, name: &str) -> Option { + self.by_name.get(name).copied() + } + + /// The merged column whose leftmost column is `(cursor, col)`. + pub(crate) fn first_at(&self, at: (i64, usize)) -> Option<&UsingCol> { + let u = &self.cols[*self.by_ref.get(&at)?]; + (u.refs.first() == Some(&at)).then_some(u) + } +} + +impl std::ops::Deref for UsingCols { + type Target = [UsingCol]; + fn deref(&self) -> &[UsingCol] { + &self.cols + } +} + +/// The post-aggregation (or materialized-row) bindings of a scope: each +/// column reference and the register holding its value, in order, with a +/// map so binding a reference costs constant time. +#[derive(Clone, Debug, Default)] +pub(crate) struct ColBindings { + list: Vec<(ColumnRef, i64)>, + /// Column -> index into `list` (its first binding). + index: HashMap, +} + +impl ColBindings { + pub(crate) fn new(list: Vec<(ColumnRef, i64)>) -> Self { + let mut index = HashMap::with_capacity(list.len()); + for (i, (c, _)) in list.iter().enumerate() { + index.entry(*c).or_insert(i); + } + ColBindings { list, index } + } + + /// The register bound to `col`. + pub(crate) fn get(&self, col: &ColumnRef) -> Option { + self.index.get(col).map(|&i| self.list[i].1) + } + + /// The bound registers, in binding order. + pub(crate) fn regs(&self) -> impl Iterator + '_ { + self.list.iter().map(|(_, r)| *r) + } + + /// The bound registers, to renumber them. + pub(crate) fn regs_mut(&mut self) -> impl Iterator { + self.list.iter_mut().map(|(_, r)| r) + } +} + /// A resolved column: where its value lives at run time. -#[derive(Clone, Copy, Debug, PartialEq, Eq)] +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] pub(crate) enum ColumnRef { /// Read column `col` from cursor `cursor`. Cursor { cursor: i64, col: usize }, @@ -132,8 +222,13 @@ pub(crate) struct NameCtx<'t> { /// instead of needing a separate post-aggregation compiler. pub agg_bindings: Vec<(String, i64)>, + /// The root variant of each bound expression in `agg_bindings`, so a + /// lookup renders a candidate to its key only when it could match. + /// Rendering every node of an N-node tree would cost O(N^2). + agg_kinds: Vec>, + /// Columns merged by USING / NATURAL JOIN. - pub using: Vec, + pub using: UsingCols, /// Post-aggregation column scope. /// @@ -142,17 +237,17 @@ pub(crate) struct NameCtx<'t> { /// a column reference must be one of these (bound to the register holding /// the key) or it is an error -- reading the cursor at that point would /// return whatever row the scan happened to stop on. - pub col_bindings: Option>, + pub col_bindings: Option, } impl<'t> NameCtx<'t> { - /// The common single-table scope, equivalent to the old - /// `(table, cursor_idx)` pair. - pub(crate) fn single(table: &'t Table, cursor: i64) -> Self { + /// A single-table scope (UPDATE / DELETE) whose table is known by + /// `alias` when given (`UPDATE d AS x ...`), else by its own name. + pub(crate) fn aliased(table: &'t Table, cursor: i64, alias: Option<&str>) -> Self { Self::multi(vec![Src { cursor, table, - refname: table.name().to_ascii_lowercase(), + refname: alias.unwrap_or(table.name()).to_ascii_lowercase(), }]) } @@ -163,7 +258,8 @@ impl<'t> NameCtx<'t> { outer: None, using_base: 0, agg_bindings: Vec::new(), - using: Vec::new(), + agg_kinds: Vec::new(), + using: UsingCols::default(), col_bindings: None, } } @@ -178,9 +274,17 @@ impl<'t> NameCtx<'t> { expr.to_string().to_ascii_lowercase() } + /// Bind `expr` (an aggregate or window call, a GROUP BY key, an output + /// alias) to the register holding its computed value. + pub(crate) fn bind_agg(&mut self, expr: &Expr, reg: i64) { + self.agg_bindings.push((Self::agg_key(expr), reg)); + self.agg_kinds.push(std::mem::discriminant(expr)); + } + /// Register holding an already-computed value for `expr`, if bound. pub(crate) fn lookup_agg(&self, expr: &Expr) -> Option { - if self.agg_bindings.is_empty() { + let kind = std::mem::discriminant(expr); + if !self.agg_kinds.contains(&kind) { return None; } let key = Self::agg_key(expr); @@ -194,16 +298,12 @@ impl<'t> NameCtx<'t> { fn bind(&self, col: ColumnRef, name: &str) -> SqawkResult { match &self.col_bindings { None => Ok(col), - Some(bound) => bound - .iter() - .find(|(c, _)| *c == col) - .map(|(_, reg)| ColumnRef::Register(*reg)) - .ok_or_else(|| { - SqawkError::InvalidSqlQuery(format!( - "'{}' must appear in GROUP BY or be used in an aggregate", - name - )) - }), + Some(bound) => bound.get(&col).map(ColumnRef::Register).ok_or_else(|| { + SqawkError::InvalidSqlQuery(format!( + "'{}' must appear in GROUP BY or be used in an aggregate", + name + )) + }), } } @@ -243,12 +343,7 @@ impl<'t> NameCtx<'t> { /// the column. fn find_here(&self, qualifier: Option<&str>, name: &str) -> SqawkResult> { let want = name.to_ascii_lowercase(); - let column_of = |src: &Src<'_>| { - src.table - .column_metadata() - .iter() - .position(|c| c.name.to_ascii_lowercase() == want) - }; + let column_of = |src: &Src<'_>| src.table.column_index(name); if let Some(qualifier) = qualifier { let qual = qualifier.to_ascii_lowercase(); let Some(src) = self.srcs.iter().find(|s| s.refname == qual) else { @@ -266,7 +361,7 @@ impl<'t> NameCtx<'t> { }; } // A USING-merged column is one column by definition, never ambiguous. - if let Some(i) = self.using.iter().position(|u| u.name == want) { + if let Some(i) = self.using.position(&want) { return Ok(Some(ColumnRef::Using(self.using_base + i))); } let mut found: Option = None; @@ -276,7 +371,7 @@ impl<'t> NameCtx<'t> { // Silently picking the first would make the result // depend on FROM order. return Err(SqawkError::InvalidSqlQuery(format!( - "Column '{}' is ambiguous across the tables in FROM", + "ambiguous column name: {}", name ))); } @@ -295,11 +390,10 @@ impl<'t> NameCtx<'t> { /// outward that was (0: this scope; more: an outer reference, which /// makes the subquery correlated). /// - /// Matching is case-insensitive. A qualified name whose qualifier no - /// scope knows falls back, on a single-table scope, to the unqualified - /// name, so `employees.age` still works when the table was given an - /// alias. With several tables a wrong qualifier is an error: the fallback - /// could silently read the same-named column of another table. + /// Matching is case-insensitive. A qualifier must name a FROM source of + /// some scope by the name it is known under there -- its alias when it + /// has one -- so `FROM employees e WHERE employees.age > 30` is an error, + /// as in SQLite. pub(crate) fn lookup( &self, qualifier: Option<&str>, @@ -318,11 +412,6 @@ impl<'t> NameCtx<'t> { scope = s.outer.as_deref(); depth += 1; } - if qualifier.is_some() && self.srcs.len() <= 1 { - if let Some(col) = self.find_here(None, name)? { - return Ok((self.bind(col, &label)?, 0)); - } - } Err(SqawkError::ColumnNotFound(label)) } @@ -334,21 +423,31 @@ impl<'t> NameCtx<'t> { } /// Resolve a `table.column` (or `alias.column`) reference in this scope - /// only, with the single-table fallback of [`Self::lookup`]. + /// only. pub(crate) fn resolve_qualified(&self, qualifier: &str, name: &str) -> SqawkResult { - if let Some(col) = self.find_here(Some(qualifier), name)? { - return Ok(col); - } - if self.srcs.len() <= 1 { - return self.resolve_unqualified(name); - } - Err(SqawkError::ColumnNotFound(format!( - "{}.{}", - qualifier, name - ))) + self.find_here(Some(qualifier), name)? + .ok_or_else(|| SqawkError::ColumnNotFound(format!("{}.{}", qualifier, name))) } } +/// The name the table declares for the column `qualifier.name` (or `name`) +/// resolves to in `ctx` or an enclosing scope -- `Id` for a reference +/// written `id` -- or `None` when it does not resolve to a table column. +fn declared_column_name(ctx: &NameCtx<'_>, qualifier: Option<&str>, name: &str) -> Option { + let (col, depth) = ctx.lookup(qualifier, name).ok()?; + let (cursor, idx) = match col { + ColumnRef::Cursor { cursor, col } => (cursor, col), + ColumnRef::Using(i) => ctx.using_col(i).refs[0], + ColumnRef::Register(_) => return None, + }; + ctx.scope_at(depth) + .srcs + .iter() + .find(|s| s.cursor == cursor) + .and_then(|s| s.table.column_metadata().get(idx)) + .map(|m| m.name.clone()) +} + /// Reject a scale or TO-datetime argument on CEIL/FLOOR. /// /// sqlparser lowers `CEIL`/`FLOOR` to [`sqlparser::ast::Expr::Ceil`] and @@ -419,6 +518,24 @@ pub struct SqlCompiler<'a> { /// While an INSERT ... SELECT or CREATE TABLE ... AS is compiled: where /// `Sink::Table` sends the query's rows. pub(crate) insert_target: Option, + + /// The text the statement was parsed from, which names unaliased + /// result columns. `None` names them by the expression's Display form. + pub(crate) source: Option>, + + /// The statement is the query of a derived table (`FROM (SELECT ...)`), + /// whose column references SQLite names as written (`id` for a column + /// declared `Id`) rather than by the declared name. + pub(crate) derived: bool, + + /// The names (lower case) of every CTE declared in the statement being + /// run. The derived-table / CTE pre-pass (`prepass.rs`) rewrites each + /// CTE reference to the CTE's materialized table and spells every other + /// table reference with one of these names `main.name`, so a one-part + /// name from this set reaching [`SqlCompiler::get_table_name`] is a + /// reference the pre-pass missed. It is an error rather than a silent + /// read of the loaded table of that name. + pub(crate) cte_names: Arc>, } /// The table an INSERT ... SELECT or CREATE TABLE ... AS writes into. @@ -451,9 +568,43 @@ impl<'a> SqlCompiler<'a> { next_cursor: FIRST_TEMP_CURSOR, next_sorter: 0, insert_target: None, + source: None, + derived: false, + cte_names: Arc::default(), } } + /// The CTE names declared in the statement (see `cte_names`). + pub fn with_cte_names(mut self, names: Arc>) -> Self { + self.cte_names = names; + self + } + + /// Name unaliased result columns by their text in `source`; `derived` + /// as the field. + pub fn with_source(mut self, source: Option>, derived: bool) -> Self { + self.source = source; + self.derived = derived; + self + } + + /// A throwaway compiler in this one's state (scope, cursor and register + /// numbering), for compiling an expression only to see what it reads: + /// its program is discarded. + pub(crate) fn scratch(&self) -> SqlCompiler<'a> { + let mut c = SqlCompiler::new(self.database, false); + c.table_map = self.table_map.clone(); + c.register_counter = self.register_counter; + c.outer_scope = self.outer_scope.clone(); + c.correlated = self.correlated.clone(); + c.next_cursor = self.next_cursor; + c.next_sorter = self.next_sorter; + c.source = self.source.clone(); + c.derived = self.derived; + c.cte_names = self.cte_names.clone(); + c + } + /// A fresh cursor id for an ephemeral table. pub(crate) fn alloc_cursor(&mut self) -> i64 { self.next_cursor += 1; @@ -633,18 +784,6 @@ impl<'a> SqlCompiler<'a> { } } - /// Convert SQL type to internal type string - pub(crate) fn sql_type_to_internal(data_type: &str) -> &'static str { - match data_type.to_uppercase().as_str() { - "INTEGER" | "INT" => "INTEGER", - "REAL" | "FLOAT" | "DOUBLE" => "REAL", - "BOOLEAN" | "BOOL" => "BOOLEAN", - _ => "TEXT", - } - } - - // Removed unused add_halt method - /// Add a comment to the program if verbose mode is enabled pub(crate) fn add_comment(&mut self, comment: &str) { if self.verbose { @@ -652,10 +791,6 @@ impl<'a> SqlCompiler<'a> { } } - // Method compile_query_direct removed - will be reimplemented when we update to a newer sqlparser version - // TODO: Add direct query compilation support - - /// Compile an SQL string into bytecode /// Compile a single already-parsed statement into its own program. /// /// Each statement gets its own program so it can be executed, and its @@ -698,7 +833,7 @@ impl<'a> SqlCompiler<'a> { /// generated bytecode for one statement. Execution goes through /// `compile_statement_program` per statement instead, so that each /// statement can observe the previous one's effects. - #[allow(dead_code)] + #[cfg(test)] pub fn compile(&mut self, sql: &str) -> SqawkResult { // Reset state for new compilation self.program = Program::new(); @@ -709,8 +844,8 @@ impl<'a> SqlCompiler<'a> { self.pending_jumps_three.clear(); // Parse the SQL statement - let dialect = HiveDialect {}; - let statements = Parser::parse_sql(&dialect, sql).map_err(SqawkError::SqlParseError)?; + let statements = super::parse::parse_sql(sql)?; + self.source = Some(Arc::new(SqlSource::new(sql))); if statements.is_empty() { return Err(SqawkError::InvalidSqlQuery( @@ -756,8 +891,6 @@ impl<'a> SqlCompiler<'a> { Ok(self.program.clone()) } - // Removed unused compile_statement_into_program method - /// Compile a single SQL statement fn compile_statement(&mut self, statement: &Statement) -> SqawkResult<()> { match statement { @@ -765,7 +898,9 @@ impl<'a> SqlCompiler<'a> { Statement::Insert(insert) => { check_insert(insert)?; let source = insert.source.as_ref().ok_or_else(|| { - SqawkError::UnsupportedSqlFeature("INSERT without a source".into()) + SqawkError::UnsupportedSqlFeature( + "INSERT without VALUES or SELECT is not supported".into(), + ) })?; // `Insert::columns` became Vec; sqawk only accepts // bare column names, so flatten each to its last segment. @@ -840,25 +975,25 @@ impl<'a> SqlCompiler<'a> { /// Expand a SELECT list into one expression per output column. /// /// `*` and `t.*` become column references, so every output column -- - /// expanded or written -- is compiled by `code_expr` alike. + /// expanded or written -- is compiled by `code_expr` alike. Columns are + /// named as SQLite names them: an alias wins; a column reference and a + /// `*` expansion give the column's own name (duplicates included); any + /// other expression gives its text exactly as written. pub(crate) fn expand_projection( &self, - projection: &[SelectItem], + select: &Select, plan: &FromPlan<'a>, ) -> SqawkResult> { - let multi = plan.srcs.len() > 1; - let star_name = |refname: &str, col: &str| { - if multi && !plan.bare_star { - format!("{}.{}", refname, col) - } else { - col.to_string() - } - }; let column = |refname: &str, col: &str| { Expr::CompoundIdentifier(vec![Ident::new(refname), Ident::new(col)]) }; + let texts = self + .source + .as_ref() + .and_then(|src| src.select_item_texts(select)); + let ctx = plan.ctx(); let mut outs = Vec::new(); - for item in projection { + for (item_no, item) in select.projection.iter().enumerate() { match item { SelectItem::Wildcard(_) => { for (si, src) in plan.srcs.iter().enumerate() { @@ -872,7 +1007,7 @@ impl<'a> SqlCompiler<'a> { }; outs.push(OutCol { expr, - name: star_name(&src.refname, &meta.name), + name: meta.name.clone(), alias: None, }); } @@ -881,7 +1016,7 @@ impl<'a> SqlCompiler<'a> { SelectItem::QualifiedWildcard(kind, _) => { let qualifier = match kind { SelectItemQualifiedWildcardKind::ObjectName(name) => { - object_name_last(name).to_ascii_lowercase() + main_table_name(name)?.to_ascii_lowercase() } SelectItemQualifiedWildcardKind::Expr(_) => { return Err(SqawkError::UnsupportedSqlFeature( @@ -889,29 +1024,27 @@ impl<'a> SqlCompiler<'a> { )) } }; - let found = plan.srcs.iter().position(|s| { - s.refname == qualifier || s.table.name().eq_ignore_ascii_case(&qualifier) - }); - let src = match found { - Some(i) => &plan.srcs[i], - // A single table answers to its own name whatever - // alias it was given. - None if !multi => &plan.srcs[0], - None => return Err(SqawkError::TableNotFound(qualifier)), - }; + let src = plan + .srcs + .iter() + .find(|s| s.refname == qualifier) + .ok_or(SqawkError::TableNotFound(qualifier))?; for meta in src.table.column_metadata() { outs.push(OutCol { expr: column(&src.refname, &meta.name), - name: star_name(&src.refname, &meta.name), + name: meta.name.clone(), alias: None, }); } } - SelectItem::UnnamedExpr(expr) => outs.push(OutCol { - expr: expr.clone(), - name: self.output_name(expr, multi), - alias: None, - }), + SelectItem::UnnamedExpr(expr) => { + let text = texts.as_ref().map(|t| t[item_no]); + outs.push(OutCol { + expr: expr.clone(), + name: self.output_name(expr, text, &ctx), + alias: None, + }) + } SelectItem::ExprWithAlias { expr, alias } => outs.push(OutCol { expr: expr.clone(), name: alias.value.clone(), @@ -929,22 +1062,55 @@ impl<'a> SqlCompiler<'a> { Ok(outs) } - /// Header for an unaliased SELECT-list expression. - /// - /// In a join a qualified column keeps its qualifier (`users.name`), as - /// join output always has; a single-table query shows the bare name. - pub(crate) fn output_name(&self, expr: &Expr, multi: bool) -> String { - match expr { - Expr::CompoundIdentifier(parts) if multi => parts - .iter() - .map(|p| p.value.as_str()) - .collect::>() - .join("."), - _ => self.get_column_name_from_expr(expr), + /// Name of an unaliased result column: a column reference (parenthesized + /// or not) is named by the column as the table declares it -- in a + /// derived table's query, as written -- anything else by `text`, its + /// source text, or failing that its Display form. + pub(crate) fn output_name(&self, expr: &Expr, text: Option<&str>, ctx: &NameCtx<'_>) -> String { + let mut e = expr; + while let Expr::Nested(inner) = e { + e = inner; + } + let reference = match e { + Expr::Identifier(ident) => Some((None, &ident.value)), + Expr::CompoundIdentifier(parts) if parts.len() == 2 => { + Some((Some(parts[0].value.as_str()), &parts[1].value)) + } + _ => None, + }; + if let Some((qualifier, name)) = reference { + if self.derived { + return name.clone(); + } + return declared_column_name(ctx, qualifier, name).unwrap_or_else(|| name.clone()); + } + match text { + Some(t) => t.to_string(), + None => expr.to_string(), } } /// Emit bytecode to load a value as a literal + /// `reg = CAST(text AS REAL)`: a real constant from its text. + fn emit_real_literal(&mut self, reg: i64, text: &str) { + self.emit( + OpCode::String, + 0, + reg, + 0, + Some(text.to_string()), + &format!("r[{}] = {}", reg, text), + ); + self.emit( + OpCode::Cast, + reg, + reg, + 0, + Some("REAL".to_string()), + &format!("Cast r[{}] to REAL", reg), + ); + } + fn emit_value_literal(&mut self, reg: i64, value: &TableValue) -> SqawkResult<()> { match value { TableValue::Integer(i) => { @@ -958,24 +1124,14 @@ impl<'a> SqlCompiler<'a> { ); } TableValue::Float(f) => { - // Store float as string and use String opcode, then cast - self.emit( - OpCode::String, - 0, - reg, - 0, - Some(f.to_string()), - &format!("r[{}] = {}", reg, f), - ); - // Cast to float - self.emit( - OpCode::Cast, - reg, - reg, - 0, - Some("REAL".to_string()), - &format!("Cast r[{}] to REAL", reg), - ); + // Rust spells infinity `inf`, which CAST reads as 0; `1e999` + // reads back as infinity (a literal such as 1e999 is one). + let text = if f.is_infinite() { + if *f > 0.0 { "1e999" } else { "-1e999" }.to_string() + } else { + f.to_string() + }; + self.emit_real_literal(reg, &text); } TableValue::String(s) => { self.emit( @@ -1251,9 +1407,20 @@ impl<'a> SqlCompiler<'a> { ctx: &NameCtx<'a>, dest: i64, ) -> SqawkResult<()> { - let def = func::lookup(name).ok_or_else(|| { - SqawkError::UnsupportedSqlFeature(format!("Unsupported function: {}", name)) - })?; + let refs: Vec<&Expr> = args.iter().collect(); + self.emit_call_refs(name, &refs, ctx, dest) + } + + /// [`Self::emit_call`] over borrowed arguments. + pub(crate) fn emit_call_refs( + &mut self, + name: &str, + args: &[&Expr], + ctx: &NameCtx<'a>, + dest: i64, + ) -> SqawkResult<()> { + let def = func::lookup(name) + .ok_or_else(|| SqawkError::InvalidSqlQuery(format!("no such function: {}", name)))?; func::check_arity(def, args.len())?; let base = self.allocate_registers(args.len().max(1)); for (i, arg) in args.iter().enumerate() { @@ -1379,6 +1546,18 @@ impl<'a> SqlCompiler<'a> { dest: i64, ) -> SqawkResult<()> { let name = Self::get_function_name(func); + // An aggregate somewhere only a scalar can be (WHERE, GROUP BY, an + // aggregate's own argument). Checked before the arguments, so + // `COUNT(*)` gets this message too. One-argument MIN/MAX is the + // aggregate; with more arguments they are scalar functions. + if AggregateFunction::is_aggregate_name(&name) + && (func::lookup(&name).is_none() || func_args(func).len() == 1) + { + return Err(SqawkError::InvalidSqlQuery(format!( + "misuse of aggregate function {}()", + name + ))); + } let args = Self::scalar_call_args(func)?; let arity = |min: usize, max: usize| -> SqawkResult<()> { if args.len() < min || args.len() > max { @@ -1407,13 +1586,8 @@ impl<'a> SqlCompiler<'a> { self.compile_iif(&args, ctx, dest) } _ if func::lookup(&name).is_some() => self.emit_call(&name, &args, ctx, dest), - // An aggregate somewhere only a scalar can be (WHERE, GROUP BY, - // an aggregate's own argument). - _ if AggregateFunction::is_aggregate_name(&name) => Err(SqawkError::InvalidSqlQuery( - format!("misuse of aggregate function {}()", name), - )), - _ => Err(SqawkError::UnsupportedSqlFeature(format!( - "Unsupported function: {}", + _ => Err(SqawkError::InvalidSqlQuery(format!( + "no such function: {}", name ))), } @@ -1499,7 +1673,13 @@ impl<'a> SqlCompiler<'a> { Value::Number(n, _) => match n.parse::() { Ok(i) => TableValue::Integer(i), Err(_) => match n.parse::() { - Ok(f) => TableValue::Float(f), + // A real literal is read from its own text by + // CAST, i.e. by SQLite's text-to-real reading + // (arith::atof), as SQLite reads it. + Ok(_) => { + self.emit_real_literal(dest, n); + return Ok(dest); + } Err(_) => TableValue::String(n.clone().into()), }, }, @@ -1511,9 +1691,15 @@ impl<'a> SqlCompiler<'a> { } Value::Boolean(b) => TableValue::Boolean(*b), Value::Null => TableValue::Null, - // Blobs, placeholders, dollar-quoted and other dialect - // literals used to become their SQL text -- X'00' was - // the four-character string "X'00'". + // `X'00'` (a hex literal `0x10` is an integer by now, + // see parse.rs). + Value::HexStringLiteral(_) => { + return Err(SqawkError::UnsupportedSqlFeature( + "blob literals are not supported".into(), + )) + } + // Placeholders, dollar-quoted and other dialect + // literals used to become their SQL text. other => { return Err(SqawkError::UnsupportedSqlFeature(format!( "Literal {} is not supported", @@ -1531,7 +1717,34 @@ impl<'a> SqlCompiler<'a> { op: BinaryOperator::StringConcat, right, } => { - self.emit_call("||", &[(**left).clone(), (**right).clone()], ctx, dest)?; + self.emit_call_refs("||", &[left, right], ctx, dest)?; + } + + // Bitwise operators (SQLite's OP_BitAnd / BitOr / ShiftLeft / + // ShiftRight / BitNot), run as the operator entries of the + // function table. + Expr::BinaryOp { + left, + op: + op @ (BinaryOperator::BitwiseAnd + | BinaryOperator::BitwiseOr + | BinaryOperator::PGBitwiseShiftLeft + | BinaryOperator::PGBitwiseShiftRight), + right, + } => { + let name = match op { + BinaryOperator::BitwiseAnd => "&", + BinaryOperator::BitwiseOr => "|", + BinaryOperator::PGBitwiseShiftLeft => "<<", + _ => ">>", + }; + self.emit_call_refs(name, &[left, right], ctx, dest)?; + } + Expr::UnaryOp { + op: UnaryOperator::BitwiseNot, + expr: inner, + } => { + self.emit_call_refs("~", &[inner], ctx, dest)?; } // Logical AND / OR with SQL three-valued logic: both operands are @@ -1697,6 +1910,13 @@ impl<'a> SqlCompiler<'a> { .. }) = &**inner { + // Only a hex literal (0x8000000000000000) becomes a + // number token with a sign; its negation does not fit. + if n == "-9223372036854775808" { + return Err(SqawkError::InvalidSqlQuery( + "hex literal too big: -0x8000000000000000".into(), + )); + } if let Ok(i) = format!("-{}", n).parse::() { self.emit_value_literal(dest, &TableValue::Integer(i))?; return Ok(dest); @@ -1981,12 +2201,21 @@ impl<'a> SqlCompiler<'a> { if name.0.is_empty() { return Err(SqawkError::InvalidSqlQuery("Empty table name".to_string())); } + if let [part] = name.0.as_slice() { + if let Some(ident) = part.as_ident() { + if self.cte_names.contains(&ident.value.to_ascii_lowercase()) { + return Err(SqawkError::VmError(format!( + "internal error: unresolved CTE {}", + ident.value + ))); + } + } + } - // Use the last part of the object name as the table name - // (schemas and catalogs are ignored for now). Table names match + // `t` or `main.t` (see `main_table_name`). Table names match // case-insensitively, as SQL identifiers do: `FROM EMP` finds the // table loaded from emp.csv. An exact match wins. - let wanted = object_name_last(name); + let wanted = main_table_name(name)?; if self.database.has_table(&wanted) { return Ok(wanted); } @@ -2001,64 +2230,6 @@ impl<'a> SqlCompiler<'a> { } } - pub(crate) fn get_column_name_from_expr(&self, expr: &Expr) -> String { - match expr { - Expr::Identifier(ident) => ident.value.clone(), - Expr::CompoundIdentifier(parts) => { - parts.last().map(|p| p.value.clone()).unwrap_or_default() - } - Expr::Function(func) => { - // Aggregate and window functions are named by the function - // alone ("COUNT" not "COUNT(*)", "ROW_NUMBER" for a window - // call); other functions include arguments for clarity. - let func_name = func.name.to_string().to_uppercase(); - if func.over.is_some() - || AggregateFunction::is_aggregate_call(&func_name, func_args(func).len()) - { - func_name - } else if let Some(FunctionArg::Unnamed(FunctionArgExpr::Wildcard)) = - func_args(func).first() - { - format!("{}(*)", func_name) - } else if let Some(FunctionArg::Unnamed(FunctionArgExpr::Expr(arg_expr))) = - func_args(func).first() - { - // ROUND's precision has to appear in the name, or - // `SELECT ROUND(a, 1), ROUND(a, 3)` yields two columns - // both called `ROUND(a)` and the second is unaddressable. - // The precision is rendered from the AST rather than named - // recursively, so a literal shows as `1` not `expr`. - if func_name == "ROUND" { - if let Some(FunctionArg::Unnamed(FunctionArgExpr::Expr(prec))) = - func_args(func).get(1) - { - return format!( - "{}({}, {})", - func_name, - self.get_column_name_from_expr(arg_expr), - prec - ); - } - } - format!( - "{}({})", - func_name, - self.get_column_name_from_expr(arg_expr) - ) - } else { - format!("{}()", func_name) - } - } - // SUBSTR/SUBSTRING is its own Expr variant rather than a Function, - // so it needs an arm here to keep the descriptive column name - // (`SUBSTR(name)`) instead of falling through to `expr`. - Expr::Substring { expr, .. } => { - format!("SUBSTR({})", self.get_column_name_from_expr(expr)) - } - _ => "expr".to_string(), - } - } - /// Infer the data type of an expression pub(crate) fn infer_expr_type(&self, expr: &Expr, ctx: &NameCtx<'_>) -> DataType { let column_type = |qualifier: Option<&str>, name: &str| -> DataType { @@ -2168,6 +2339,10 @@ impl<'a> SqlCompiler<'a> { | BinaryOperator::And | BinaryOperator::Or => DataType::Boolean, BinaryOperator::StringConcat => DataType::Text, + BinaryOperator::BitwiseAnd + | BinaryOperator::BitwiseOr + | BinaryOperator::PGBitwiseShiftLeft + | BinaryOperator::PGBitwiseShiftRight => DataType::Integer, _ => DataType::Text, } } diff --git a/src/vm/compiler_aggregate.rs b/src/vm/compiler_aggregate.rs index 0a09f22..b5d2d5f 100644 --- a/src/vm/compiler_aggregate.rs +++ b/src/vm/compiler_aggregate.rs @@ -11,16 +11,13 @@ //! bound to the register holding its value, which is what makes `SUM(x) + 1`, //! `ROUND(AVG(x), 1)`, `UPPER(key)` and arbitrary HAVING expressions work. -use sqlparser::ast::{ - Expr, FunctionArg, FunctionArgExpr, FunctionArguments, OrderByExpr, Select, Value, - ValueWithSpan, -}; +use sqlparser::ast::{Expr, FunctionArg, FunctionArgExpr, FunctionArguments, OrderByExpr, Select}; use super::ast_compat::{func_args, group_by_exprs}; use super::bytecode::{OpCode, AGG_DISTINCT, AGG_SEP, AGG_TYPE_MASK}; use super::compiler::{NameCtx, OutCol, SqlCompiler}; -use super::compiler_from::{expr_children, FromPlan}; -use super::compiler_select::{Dest, OutputStage}; +use super::compiler_from::{expr_children, reads_only_outer, FromPlan}; +use super::compiler_select::{term_position, Dest, OutputStage}; use super::compiler_window::{window_call_children, WindowStage}; use crate::aggregate::AggregateFunction; use crate::error::{SqawkError, SqawkResult}; @@ -185,22 +182,12 @@ impl<'a> SqlCompiler<'a> { // GROUP BY keys. `GROUP BY 2` names the second output column, and a // name that is no column but is a SELECT alias names that column. let mut keys: Vec = Vec::new(); - for g in group_by_exprs(&select.group_by) { + for (i, g) in group_by_exprs(&select.group_by).iter().enumerate() { + if let Some(j) = term_position(g, i, "GROUP", outs.len())? { + keys.push(outs[j].expr.clone()); + continue; + } let key = match g { - Expr::Value(ValueWithSpan { - value: Value::Number(n, _), - .. - }) => { - let pos: usize = n.parse().unwrap_or(0); - if pos == 0 || pos > outs.len() { - return Err(SqawkError::InvalidSqlQuery(format!( - "GROUP BY position {} is out of range (1..{})", - n, - outs.len() - ))); - } - outs[pos - 1].expr.clone() - } Expr::Identifier(ident) if ctx.resolve_unqualified(&ident.value).is_err() => { match outs.iter().find(|o| { o.alias @@ -239,6 +226,18 @@ impl<'a> SqlCompiler<'a> { .into_iter() .map(Self::describe_aggregate) .collect::>>()?; + // SQLite evaluates an aggregate whose arguments read only columns of + // an enclosing query in that enclosing query, turning it into an + // aggregate query. sqawk aggregates where the call is written, which + // would silently give a different answer, so the call is rejected. + for agg in &aggs { + if reads_only_outer(&agg.args, &ctx) { + return Err(SqawkError::UnsupportedSqlFeature(format!( + "aggregate of outer-query columns is not supported: {}", + agg.call + ))); + } + } let group = Group { plan, having: select.having.as_ref(), @@ -477,13 +476,12 @@ impl<'a> SqlCompiler<'a> { // registers; any other column reference is an error. let mut ctx = g.plan.ctx(); for (j, agg) in g.aggs.iter().enumerate() { - ctx.agg_bindings - .push((NameCtx::agg_key(&agg.call), fin + j as i64)); + ctx.bind_agg(&agg.call, fin + j as i64); } let mut cols = Vec::new(); for (i, k) in g.keys.iter().enumerate() { let reg = key_reg + i as i64; - ctx.agg_bindings.push((NameCtx::agg_key(k), reg)); + ctx.bind_agg(k, reg); let col = match k { Expr::Identifier(ident) => ctx.resolve_unqualified(&ident.value).ok(), Expr::CompoundIdentifier(parts) if parts.len() >= 2 => ctx @@ -495,7 +493,7 @@ impl<'a> SqlCompiler<'a> { cols.push((col, reg)); } } - ctx.col_bindings = Some(cols); + ctx.col_bindings = Some(super::compiler::ColBindings::new(cols)); match ws { Some(ws) => self.emit_window_row(ws, &ctx, g.having), None => self.emit_stage_row(g.stage, g.outs, &ctx, g.having), diff --git a/src/vm/compiler_check.rs b/src/vm/compiler_check.rs index d77cf56..5d4c2a1 100644 --- a/src/vm/compiler_check.rs +++ b/src/vm/compiler_check.rs @@ -9,15 +9,16 @@ //! "Unsupported" error before a single row is touched. //! //! The checks destructure each node EXHAUSTIVELY (no `..`), so a field added -//! by a future sqlparser release fails to compile here until someone decides +//! by a newer sqlparser release fails to compile here until someone decides //! whether sqawk honours it. use sqlparser::ast::{ - CreateTable, Delete, FromTable, GroupByExpr, HiveDistributionStyle, HiveFormat, Insert, - LimitClause, OrderBy, OrderByExpr, OrderByOptions, Query, Select, SelectFlavor, TableAlias, - TableFactor, TableWithJoins, Update, + CreateTable, CreateTableOptions, Delete, FileFormat, FromTable, GroupByExpr, + HiveDistributionStyle, HiveFormat, HiveIOFormat, Insert, LimitClause, OrderBy, OrderByExpr, + OrderByOptions, Query, Select, SelectFlavor, TableAlias, TableFactor, TableWithJoins, Update, }; +use super::ast_compat::sql_option_key_value; use crate::error::{SqawkError, SqawkResult}; fn unsupported(what: &str) -> SqawkResult { @@ -36,6 +37,76 @@ fn reject(present: bool, what: &str) -> SqawkResult<()> { } } +/// A transaction-control statement sqawk implements. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) enum TransactionControl { + Begin, + Commit, + Rollback, +} + +/// Classify `statement` as BEGIN / COMMIT / ROLLBACK, `None` for any other +/// statement. +/// +/// Accepted, as in SQLite: `BEGIN [DEFERRED | IMMEDIATE | EXCLUSIVE] +/// [TRANSACTION]` (the modifiers only choose locks, and sqawk has no +/// concurrent connections, so all three mean plain BEGIN), `COMMIT` / +/// `END [TRANSACTION]` and `ROLLBACK [TRANSACTION]`. Rejected: savepoints +/// (`SAVEPOINT`, `RELEASE`, `ROLLBACK TO`), transaction modes (`ISOLATION +/// LEVEL`, `READ ONLY`, ...), `AND [NO] CHAIN`, and `BEGIN ... END` blocks. +pub(crate) fn transaction_control( + statement: &sqlparser::ast::Statement, +) -> SqawkResult> { + use sqlparser::ast::{Statement, TransactionModifier}; + let lock_modifier = |m: &Option| { + matches!( + m, + None | Some( + TransactionModifier::Deferred + | TransactionModifier::Immediate + | TransactionModifier::Exclusive + ) + ) + }; + match statement { + Statement::StartTransaction { + modes, + begin, + transaction: _, + modifier, + statements, + exception, + has_end_keyword, + } => { + reject(!*begin, "START TRANSACTION (use BEGIN)")?; + reject(!modes.is_empty(), "Transaction modes")?; + reject(!lock_modifier(modifier), "BEGIN TRY / BEGIN CATCH")?; + reject( + !statements.is_empty() || exception.is_some() || *has_end_keyword, + "BEGIN ... END blocks", + )?; + Ok(Some(TransactionControl::Begin)) + } + Statement::Commit { + chain, + end: _, + modifier, + } => { + reject(*chain, "COMMIT AND CHAIN")?; + reject(!lock_modifier(modifier), "END TRY / END CATCH")?; + Ok(Some(TransactionControl::Commit)) + } + Statement::Rollback { chain, savepoint } => { + reject(savepoint.is_some(), "ROLLBACK TO (savepoints)")?; + reject(*chain, "ROLLBACK AND CHAIN")?; + Ok(Some(TransactionControl::Rollback)) + } + Statement::Savepoint { .. } => unsupported("SAVEPOINT"), + Statement::ReleaseSavepoint { .. } => unsupported("RELEASE (savepoints)"), + _ => Ok(None), + } +} + /// INSERT: only `INSERT INTO t [(cols)] VALUES ... | SELECT ...`. pub(crate) fn check_insert(insert: &Insert) -> SqawkResult<()> { let Insert { @@ -179,7 +250,12 @@ pub(crate) fn check_query(query: &Query) -> SqawkResult<()> { format_clause, pipe_operators, } = query; - reject(with.is_some(), "WITH (common table expressions)")?; + // The pre-pass (`prepass.rs`) resolves every WITH and removes it. + if with.is_some() { + return Err(SqawkError::VmError( + "internal error: unresolved WITH".to_string(), + )); + } reject(fetch.is_some(), "FETCH FIRST")?; reject(!locks.is_empty(), "FOR UPDATE / FOR SHARE")?; reject(for_clause.is_some(), "FOR XML / FOR JSON")?; @@ -323,48 +399,237 @@ pub(crate) fn check_order_by_expr(ob: &OrderByExpr) -> SqawkResult<()> { reject(with_fill.is_some(), "ORDER BY ... WITH FILL") } -/// CREATE TABLE: column names and types, `LOCATION`, `STORED AS TEXTFILE`, -/// `WITH (...)` options, or `AS query`. Constraints are not enforced by -/// sqawk, so declaring one is an error rather than a silent no-op. +/// CREATE TABLE: column names and types, `LOCATION`, `STORED AS TEXTFILE` +/// and `WITH (DELIMITER='...', HEADER=true|false, COMMENTS=true|false)`; or +/// `AS query` with none of those. +/// `TEMPORARY` is accepted for a table without `LOCATION` (every such table +/// lives only for the session). Constraints are not enforced by sqawk, so +/// declaring one is an error rather than a silent no-op; so is every other +/// clause. pub(crate) fn check_create_table(create: &CreateTable) -> SqawkResult<()> { - reject(create.or_replace, "CREATE OR REPLACE TABLE")?; - reject(create.if_not_exists, "CREATE TABLE IF NOT EXISTS")?; - reject(create.external, "CREATE EXTERNAL TABLE")?; - reject(!create.constraints.is_empty(), "Table constraints")?; + let CreateTable { + or_replace, + temporary, + external, + dynamic, + global, + if_not_exists, + transient, + volatile, + iceberg, + snapshot, + name: _, + columns, + constraints, + hive_distribution, + hive_formats, + table_options, + file_format, + location, + query, + without_rowid, + like, + clone, + version, + comment, + on_commit, + on_cluster, + primary_key, + order_by, + partition_by, + cluster_by, + clustered_by, + inherits, + partition_of, + for_values, + strict, + copy_grants, + enable_schema_evolution, + change_tracking, + data_retention_time_in_days, + max_data_extension_time_in_days, + default_ddl_collation, + with_aggregation_policy, + with_row_access_policy, + with_storage_lifecycle_policy, + with_tags, + external_volume, + base_location, + catalog, + catalog_sync, + storage_serialization_policy, + target_lag, + warehouse, + refresh_mode, + initialize, + require_user, + diststyle, + distkey, + sortkey, + backup, + } = create; + reject(*or_replace, "CREATE OR REPLACE TABLE")?; + reject(*if_not_exists, "CREATE TABLE IF NOT EXISTS")?; + reject(*external, "CREATE EXTERNAL TABLE")?; reject( - create.columns.iter().any(|c| !c.options.is_empty()), - "Column constraints (NOT NULL, PRIMARY KEY, DEFAULT, ...)", + *dynamic + || global.is_some() + || *transient + || *volatile + || *iceberg + || *snapshot + || *require_user, + "CREATE TABLE modifiers (GLOBAL, TRANSIENT, VOLATILE, ...)", )?; - reject(create.like.is_some(), "CREATE TABLE ... LIKE")?; - reject(create.clone.is_some(), "CREATE TABLE ... CLONE")?; - reject(create.without_rowid, "WITHOUT ROWID")?; - reject(create.strict, "STRICT tables")?; - reject(create.primary_key.is_some(), "PRIMARY KEY")?; - reject(create.order_by.is_some(), "CREATE TABLE ... ORDER BY")?; + reject(!constraints.is_empty(), "Table constraints")?; reject( - create.partition_by.is_some(), - "CREATE TABLE ... PARTITION BY", + columns.iter().any(|c| !c.options.is_empty()), + "Column constraints (NOT NULL, PRIMARY KEY, DEFAULT, ...)", )?; + reject(like.is_some(), "CREATE TABLE ... LIKE")?; + reject(clone.is_some(), "CREATE TABLE ... CLONE")?; + reject(version.is_some(), "CREATE TABLE ... AT / BEFORE")?; + reject(comment.is_some(), "Table comments")?; + reject(on_commit.is_some(), "ON COMMIT")?; + reject(on_cluster.is_some(), "ON CLUSTER")?; + reject(*without_rowid, "WITHOUT ROWID")?; + reject(*strict, "STRICT tables")?; + reject(primary_key.is_some(), "PRIMARY KEY")?; + reject(order_by.is_some(), "CREATE TABLE ... ORDER BY")?; + reject(partition_by.is_some(), "CREATE TABLE ... PARTITION BY")?; reject( - create.cluster_by.is_some() || create.clustered_by.is_some(), + cluster_by.is_some() || clustered_by.is_some(), "CREATE TABLE ... CLUSTER BY", )?; - reject(create.inherits.is_some(), "INHERITS")?; + reject(inherits.is_some(), "INHERITS")?; + reject( + partition_of.is_some() || for_values.is_some(), + "PARTITION OF", + )?; + reject( + *copy_grants + || enable_schema_evolution.is_some() + || change_tracking.is_some() + || data_retention_time_in_days.is_some() + || max_data_extension_time_in_days.is_some() + || default_ddl_collation.is_some() + || with_aggregation_policy.is_some() + || with_row_access_policy.is_some() + || with_storage_lifecycle_policy.is_some() + || with_tags.is_some() + || external_volume.is_some() + || base_location.is_some() + || catalog.is_some() + || catalog_sync.is_some() + || storage_serialization_policy.is_some() + || target_lag.is_some() + || warehouse.is_some() + || refresh_mode.is_some() + || initialize.is_some(), + "Warehouse table properties", + )?; reject( - !matches!(create.hive_distribution, HiveDistributionStyle::NONE), + diststyle.is_some() || distkey.is_some() || sortkey.is_some() || backup.is_some(), + "DISTSTYLE / DISTKEY / SORTKEY / BACKUP", + )?; + reject( + !matches!(hive_distribution, HiveDistributionStyle::NONE), "PARTITIONED BY / SKEWED BY", )?; + if let Some(format) = file_format { + check_file_format(format)?; + } + let mut has_location = location.is_some(); if let Some(HiveFormat { row_format, serde_properties, - storage: _, - location: _, - }) = &create.hive_formats + storage, + location, + }) = hive_formats { reject( row_format.is_some() || serde_properties.is_some(), "ROW FORMAT (use WITH (DELIMITER='...'))", )?; + match storage { + None => {} + Some(HiveIOFormat::FileFormat { format }) => check_file_format(format)?, + Some(HiveIOFormat::Using { format }) => { + unsupported(&format!("USING {}", format))?; + } + Some(HiveIOFormat::IOF { .. }) => { + unsupported("STORED AS INPUTFORMAT ... OUTPUTFORMAT")? + } + } + has_location |= location.is_some(); + } + let options = match table_options { + CreateTableOptions::None => &[][..], + CreateTableOptions::With(options) => options.as_slice(), + CreateTableOptions::Options(_) + | CreateTableOptions::Plain(_) + | CreateTableOptions::TableProperties(_) => { + return unsupported("Table options other than WITH (DELIMITER / HEADER / COMMENTS)") + } + }; + for option in options { + match sql_option_key_value(option) { + Some((key, _)) if key.eq_ignore_ascii_case("delimiter") => {} + Some((key, value)) + if key.eq_ignore_ascii_case("header") || key.eq_ignore_ascii_case("comments") => + { + if option_flag(&value).is_none() { + return Err(SqawkError::InvalidSqlQuery(format!( + "CREATE TABLE option {} must be true or false, not '{}'", + key.to_uppercase(), + value + ))); + } + } + Some((key, _)) => unsupported(&format!("The CREATE TABLE option '{}'", key))?, + None => unsupported(&format!("The CREATE TABLE option {}", option))?, + } + } + reject(*temporary && has_location, "TEMPORARY table with LOCATION")?; + reject( + !has_location + && options.iter().any(|o| { + sql_option_key_value(o).is_some_and(|(key, _)| { + key.eq_ignore_ascii_case("header") || key.eq_ignore_ascii_case("comments") + }) + }), + "WITH (HEADER / COMMENTS) without LOCATION", + )?; + if query.is_some() { + reject(!columns.is_empty(), "A column list on CREATE TABLE ... AS")?; + reject( + has_location || !options.is_empty() || file_format.is_some(), + "LOCATION / STORED AS / WITH on CREATE TABLE ... AS", + )?; + if let Some(HiveFormat { storage, .. }) = hive_formats { + reject(storage.is_some(), "STORED AS on CREATE TABLE ... AS")?; + } } Ok(()) } + +/// The value of a boolean CREATE TABLE option (`HEADER`, `COMMENTS`): +/// `true` or `false`, any case, quoted or not. +pub(crate) fn option_flag(value: &str) -> Option { + if value.eq_ignore_ascii_case("true") { + Some(true) + } else if value.eq_ignore_ascii_case("false") { + Some(false) + } else { + None + } +} + +/// `STORED AS `: only TEXTFILE, the delimited text sqawk reads and +/// writes. +fn check_file_format(format: &FileFormat) -> SqawkResult<()> { + match format { + FileFormat::TEXTFILE => Ok(()), + other => unsupported(&format!("STORED AS {}", other)), + } +} diff --git a/src/vm/compiler_ddl.rs b/src/vm/compiler_ddl.rs index 859cabd..759e911 100644 --- a/src/vm/compiler_ddl.rs +++ b/src/vm/compiler_ddl.rs @@ -5,11 +5,12 @@ use sqlparser::ast::{AlterTableOperation, ObjectName, ObjectType}; use super::ast_compat::sql_option_key_value; -use super::bytecode::{OpCode, ResultSchema}; +use super::bytecode::{spec_escape, OpCode, ResultSchema}; use super::compiler::{InsertTarget, SqlCompiler}; +use super::compiler_check::option_flag; use super::compiler_select::Sink; use crate::error::{SqawkError, SqawkResult}; -use crate::table::DataType; +use crate::table::unique_column_names; impl<'a> SqlCompiler<'a> { /// Compile a DROP statement (DROP TABLE) @@ -19,7 +20,7 @@ impl<'a> SqlCompiler<'a> { names: &[ObjectName], if_exists: bool, ) -> SqawkResult<()> { - // Only support DROP TABLE for now + // DROP TABLE is the only DROP form sqawk implements. if *object_type != ObjectType::Table { return Err(SqawkError::UnsupportedSqlFeature(format!( "DROP {:?} is not supported, only DROP TABLE", @@ -78,9 +79,24 @@ impl<'a> SqlCompiler<'a> { match operation { AlterTableOperation::AddColumn { column_def, .. } => { let col_name = column_def.name.value.clone(); - let col_type = Self::sql_type_to_internal(&column_def.data_type.to_string()); - - let spec = format!("{}:{}:{}", table_name, col_name, col_type); + if self + .database + .get_table(&table_name)? + .column_index(&col_name) + .is_some() + { + return Err(SqawkError::InvalidSqlQuery(format!( + "duplicate column name: {}", + col_name + ))); + } + // The declared type name is recorded as written. + let spec = format!( + "{}:{}:{}", + spec_escape(&table_name), + spec_escape(&col_name), + spec_escape(&column_def.data_type.to_string()) + ); self.emit( OpCode::AlterTableAdd, @@ -174,15 +190,13 @@ impl<'a> SqlCompiler<'a> { let schema = schema?; self.program.set_result_schema(ResultSchema::new()); - let mut spec = table_name.clone(); - for col in &schema.columns { - let type_str = match col.data_type { - DataType::Integer => "INTEGER", - DataType::Float => "REAL", - DataType::Boolean => "BOOLEAN", - _ => "TEXT", - }; - spec.push_str(&format!(":{}:{}", col.name, type_str)); + // The new table's columns are named as SQLite names them: the + // query's result names, made unique (`id, id:1`). + let mut names: Vec = schema.columns.iter().map(|c| c.name.clone()).collect(); + unique_column_names(&mut names); + let mut spec = spec_escape(&table_name); + for (col, name) in schema.columns.iter().zip(&names) { + spec.push_str(&format!(":{}:{}", spec_escape(name), col.data_type)); } spec.push_str("||"); self.program.instructions[create].p4 = Some(spec.into()); @@ -201,14 +215,28 @@ impl<'a> SqlCompiler<'a> { // Extract table name let table_name = self.get_table_name(name)?; + // Column names are case-insensitive: `x` and `X` are one column. + for (i, col) in columns.iter().enumerate() { + let name = &col.name.value; + if columns[..i] + .iter() + .any(|c| c.name.value.eq_ignore_ascii_case(name)) + { + return Err(SqawkError::InvalidSqlQuery(format!( + "duplicate column name: {}", + name + ))); + } + } + // Build column specification: "table_name:col1:type1:col2:type2:..." - let mut spec = table_name.clone(); + // Each type is the declared name as written; it is not enforced. + let mut spec = spec_escape(&table_name); for col in columns { spec.push(':'); - spec.push_str(&col.name.value); + spec.push_str(&spec_escape(&col.name.value)); spec.push(':'); - let type_str = Self::sql_type_to_internal(&col.data_type.to_string()); - spec.push_str(type_str); + spec.push_str(&spec_escape(&col.data_type.to_string())); } // Get location from hive_formats or direct location @@ -217,27 +245,35 @@ impl<'a> SqlCompiler<'a> { .and_then(|hf| hf.location.clone()) .or_else(|| location.clone()); - // Extract delimiter from WITH options - let delimiter = with_options.iter().find_map(|opt| { - let (key, value) = sql_option_key_value(opt)?; - if key.to_lowercase() == "delimiter" { - // sql_option_key_value unwraps the literal, so no quote - // stripping is needed -- and stripping only single quotes left - // double-quoted values carrying their quotes. - return Some(value); - } - None - }); - - // Add file path and delimiter to spec: "...|filepath|delimiter" + // The WITH options (validated by check_create_table). sql_option_key_value + // unwraps the literal, so no quote stripping is needed. + let option = |name: &str| { + with_options.iter().find_map(|opt| { + let (key, value) = sql_option_key_value(opt)?; + key.eq_ignore_ascii_case(name).then_some(value) + }) + }; + let delimiter = option("delimiter"); + let flag = |name: &str, default: bool| { + option(name) + .and_then(|v| option_flag(&v)) + .unwrap_or(default) + }; + let header = flag("header", true); + let comments = flag("comments", false); + + // Add file path, delimiter and flags to spec: + // "...|filepath|delimiter|header|comments" spec.push('|'); if let Some(ref fp) = file_path { - spec.push_str(fp); + spec.push_str(&spec_escape(fp)); } spec.push('|'); if let Some(ref d) = delimiter { - spec.push_str(d); + spec.push_str(&spec_escape(d)); } + spec.push_str(if header { "|1" } else { "|0" }); + spec.push_str(if comments { "|1" } else { "|0" }); // Generate Init let init_addr = self.program.len(); diff --git a/src/vm/compiler_dml.rs b/src/vm/compiler_dml.rs index c48b36d..c5c4aa7 100644 --- a/src/vm/compiler_dml.rs +++ b/src/vm/compiler_dml.rs @@ -163,8 +163,8 @@ impl<'a> SqlCompiler<'a> { // Get table name let table_with_joins = &from[0]; - let table_name = match &table_with_joins.relation { - TableFactor::Table { name, .. } => self.get_table_name(name)?, + let (table_name, alias) = match &table_with_joins.relation { + TableFactor::Table { name, alias, .. } => (self.get_table_name(name)?, alias.as_ref()), _ => { return Err(SqawkError::UnsupportedSqlFeature( "Only simple table references are supported in DELETE".to_string(), @@ -207,7 +207,9 @@ impl<'a> SqlCompiler<'a> { // If there's a WHERE clause, check if row matches if let Some(where_expr) = selection { // Compile WHERE condition using table context - let cond_reg = self.code_expr(where_expr, &NameCtx::single(table, cursor_idx), None)?; + let alias = alias.map(|a| a.name.value.as_str()); + let ctx = NameCtx::aliased(table, cursor_idx, alias); + let cond_reg = self.code_expr(where_expr, &ctx, None)?; // If condition is false (0), skip to next row let skip_addr = self.program.len(); @@ -277,8 +279,11 @@ impl<'a> SqlCompiler<'a> { selection: Option<&Expr>, ) -> SqawkResult<()> { // Get table name - let table_name = match &table.relation { - TableFactor::Table { name, .. } => self.get_table_name(name)?, + let (table_name, alias) = match &table.relation { + TableFactor::Table { name, alias, .. } => ( + self.get_table_name(name)?, + alias.as_ref().map(|a| a.name.value.as_str()), + ), _ => { return Err(SqawkError::UnsupportedSqlFeature( "Only simple table references are supported in UPDATE".to_string(), @@ -343,7 +348,8 @@ impl<'a> SqlCompiler<'a> { // If there's a WHERE clause, check if row matches let skip_addr = if let Some(where_expr) = selection { // Compile WHERE condition using table context - let cond_reg = self.code_expr(where_expr, &NameCtx::single(table_ref, 0), None)?; + let cond_reg = + self.code_expr(where_expr, &NameCtx::aliased(table_ref, 0, alias), None)?; // If condition is false (0), skip to next row let addr = self.program.len(); @@ -369,7 +375,7 @@ impl<'a> SqlCompiler<'a> { // Reading through a Column op against the cursor yields the ORIGINAL // row value, which is the required semantics -- every RHS is evaluated // against the row as it was before any assignment in this statement. - let update_ctx = NameCtx::single(table_ref, 0); + let update_ctx = NameCtx::aliased(table_ref, 0, alias); for (col_idx, expr) in &assignment_map { self.code_expr(expr, &update_ctx, Some((base_reg + *col_idx) as i64))?; } diff --git a/src/vm/compiler_from.rs b/src/vm/compiler_from.rs index bba20ba..353b125 100644 --- a/src/vm/compiler_from.rs +++ b/src/vm/compiler_from.rs @@ -13,7 +13,12 @@ //! * ON conjuncts are evaluated at their own level, before that level's //! "match" point. WHERE conjuncts are evaluated after it, at the innermost //! level that has every table they mention, so the NULL-extended rows an -//! outer join produces are filtered by WHERE like any other row. +//! outer join produces are filtered by WHERE like any other row. A +//! conjunct with a subquery is placed by the columns its compiled code +//! reads (a correlated subquery's references included), so `t1.k IN +//! (SELECT ...)` filters at t1's loop. A conjunct reading no table of the +//! FROM at all is tested once, before the nest: in a correlated subquery, +//! `t1.b < t1.c` skips the scan for an outer row that fails it. //! * An outer join's preserved side owns a match flag. When an inner loop //! finishes without a match, its cursors are put on a NULL row (`NullRow`) //! and control re-enters the loop body at the match point once, so the rest @@ -29,16 +34,17 @@ //! The rest of the SELECT (projection, aggregation, sorting) is a "body" //! callback emitted at the innermost point of the nest. +use std::collections::HashSet; + use sqlparser::ast::{ BinaryOperator, Expr, FunctionArg, FunctionArgExpr, FunctionArguments, Ident, JoinConstraint, JoinOperator, ObjectName, TableFactor, TableWithJoins, }; use super::bytecode::OpCode; -use super::compiler::{ColumnRef, Label, NameCtx, SqlCompiler, Src, UsingCol}; +use super::compiler::{ColumnRef, Label, NameCtx, SqlCompiler, Src, UsingCol, UsingCols}; use crate::database::Database; use crate::error::{SqawkError, SqawkResult}; -use crate::table::Table; /// The join operator between the sources before a FROM item and the item. #[derive(Clone, Copy, Debug, PartialEq, Eq)] @@ -245,6 +251,17 @@ struct Group { end: usize, } +/// What every level of one loop nest shares (see `emit_level`). +#[derive(Clone, Copy)] +struct Levels<'r> { + /// [`FromPlan::deferred_levels`]. + deferred: &'r [bool], + /// Each outer-join group's match flag register. + match_regs: &'r [i64], + /// Where each group's NULL-extended row re-enters the nest. + match_labels: &'r [Label], +} + /// A planned FROM clause. pub(crate) struct FromPlan<'t> { /// Sources in FROM order. A statement-level query opens source `i` on @@ -257,18 +274,12 @@ pub(crate) struct FromPlan<'t> { levels: Vec, groups: Vec, /// USING / NATURAL merged columns. - pub using: Vec, + pub using: UsingCols, /// (source, column) pairs a USING merge hides from `*`. - hidden: Vec<(usize, usize)>, + hidden: HashSet<(usize, usize)>, /// The FROM clause ends in a FULL JOIN: the WHERE conjuncts, re-applied /// in the pass over unmatched right rows. full_where: Option>, - /// `*` expands to bare column names rather than `table.column`. - /// - /// Joins have always labelled `*` columns `table.column`; a USING or - /// NATURAL join instead merges the shared columns into one, and SQL names - /// its columns bare (`k,a,b`), which is what this selects. - pub bare_star: bool, /// WHERE conjuncts of a SELECT without FROM: no loop to place them in, /// so they decide whether its single row exists. const_where: Vec, @@ -302,9 +313,7 @@ impl<'t> FromPlan<'t> { /// The USING column first appearing at `(src, col)`, if any. pub(crate) fn using_at(&self, src: usize, col: usize) -> Option<&UsingCol> { - self.using - .iter() - .find(|u| u.refs.first() == Some(&(self.cursor(src), col))) + self.using.first_at((self.cursor(src), col)) } /// The level holding source `src`. @@ -320,6 +329,23 @@ impl<'t> FromPlan<'t> { self.groups.iter().find(|g| g.start <= lvl && lvl <= g.end) } + /// Per level: whether its table is opened (and its hash index built) + /// only when the nest first reaches the level, as SQLite builds an + /// automatic index there, rather than before the nest. Every level but + /// the outermost, except one a RIGHT JOIN may NULL-extend without + /// reaching (an outer-join group's levels after its first) and in a + /// FULL JOIN, whose pass over unmatched right rows reads its tables + /// after the nest. + fn deferred_levels(&self) -> Vec { + (0..self.levels.len()) + .map(|lvl| { + lvl > 0 + && self.full_where.is_none() + && !self.groups.iter().any(|g| g.start < lvl && lvl <= g.end) + }) + .collect() + } + /// The deepest level among `srcs` (level 0 for none). fn max_level(&self, srcs: &[usize]) -> usize { srcs.iter().map(|s| self.level_of(*s)).max().unwrap_or(0) @@ -378,6 +404,56 @@ fn collect_sources(expr: &Expr, ctx: &NameCtx<'_>, out: &mut Vec) -> Opti } } +impl<'a> SqlCompiler<'a> { + /// The sources of this FROM that a WHERE conjunct reads when + /// [`expr_sources`] cannot tell (a subquery -- whose correlated + /// references count -- or another node it does not analyze): found by + /// compiling the conjunct in a scratch compiler and collecting the + /// cursors its `Column` reads name. `None` when the conjunct does not + /// compile (the real compilation reports why). + fn compiled_term_sources(&self, term: &Expr, ctx: &NameCtx<'a>) -> Option> { + let mut scratch = self.scratch(); + scratch.code_expr(term, ctx, None).ok()?; + let mut out: Vec = scratch + .program + .instructions + .iter() + .filter(|inst| inst.opcode == OpCode::Column) + .filter_map(|inst| ctx.srcs.iter().position(|s| s.cursor == inst.p1)) + .collect(); + out.sort_unstable(); + out.dedup(); + Some(out) + } +} + +/// Whether `exprs` read at least one column and every column they read +/// belongs to an enclosing query. An expression that cannot be analyzed (a +/// subquery, an unresolvable name) does not count. +pub(crate) fn reads_only_outer(exprs: &[Expr], ctx: &NameCtx<'_>) -> bool { + fn walk(expr: &Expr, ctx: &NameCtx<'_>, read_any: &mut bool) -> Option<()> { + let name = match expr { + Expr::Identifier(ident) => Some((None, &ident.value)), + Expr::CompoundIdentifier(parts) if parts.len() >= 2 => Some(( + Some(parts[parts.len() - 2].value.as_str()), + &parts[parts.len() - 1].value, + )), + _ => None, + }; + if let Some((qualifier, name)) = name { + let (_, depth) = ctx.lookup(qualifier, name).ok()?; + *read_any = true; + return (depth > 0).then_some(()); + } + for child in expr_children(expr)? { + walk(child, ctx, read_any)?; + } + Some(()) + } + let mut read_any = false; + exprs.iter().all(|e| walk(e, ctx, &mut read_any).is_some()) && read_any +} + /// Whether `expr` reads a column of an enclosing query. Unanalyzable /// expressions (subqueries and the like) count as doing so. fn mentions_outer(expr: &Expr, ctx: &NameCtx<'_>) -> bool { @@ -410,13 +486,6 @@ fn column_equality(left_ref: &str, right_ref: &str, col: &str) -> Expr { } } -fn column_index_ci(table: &Table, name: &str) -> Option { - table - .column_metadata() - .iter() - .position(|c| c.name.eq_ignore_ascii_case(name)) -} - impl<'a> SqlCompiler<'a> { /// Plan a FROM clause together with the WHERE clause filtering it. pub(crate) fn plan_from( @@ -426,7 +495,7 @@ impl<'a> SqlCompiler<'a> { ) -> SqawkResult> { if self.verbose { if from.len() > 1 { - eprintln!("Processing multiple tables in FROM clause as CROSS JOINs"); + diag!("Processing multiple tables in FROM clause as CROSS JOINs"); } for join in from.iter().flat_map(|twj| &twj.joins) { if let Ok((kind, constraint)) = classify_join(&join.join_operator) { @@ -439,12 +508,12 @@ impl<'a> SqlCompiler<'a> { }; let mut title = kind.to_ascii_lowercase(); title[..1].make_ascii_uppercase(); - eprintln!("Join type: {}", title); - eprintln!("Processing {} JOIN with ON condition", kind); + diag!("Join type: {}", title); + diag!("Processing {} JOIN with ON condition", kind); } } if let Some(w) = selection { - eprintln!("WHERE comparison: {:?}", w); + diag!("WHERE comparison: {:?}", w); } } @@ -459,10 +528,9 @@ impl<'a> SqlCompiler<'a> { table_names: Vec::new(), levels: Vec::new(), groups: Vec::new(), - using: Vec::new(), - hidden: Vec::new(), + using: UsingCols::default(), + hidden: HashSet::new(), full_where: None, - bare_star: false, const_where, outer: self.outer_scope.clone().map(Box::new), }); @@ -548,10 +616,9 @@ impl<'a> SqlCompiler<'a> { } // ---- ON conjuncts per join, with USING / NATURAL lowered onto ON. - let mut using: Vec = Vec::new(); - let mut hidden: Vec<(usize, usize)> = Vec::new(); + let mut using = UsingCols::default(); + let mut hidden: HashSet<(usize, usize)> = HashSet::new(); let mut join_terms: Vec> = vec![Vec::new(); srcs.len()]; - let mut bare_star = false; for (i, (_, op)) in factors.iter().enumerate() { let Some((_, constraint)) = op else { continue }; let names: Vec = match constraint { @@ -573,19 +640,18 @@ impl<'a> SqlCompiler<'a> { .filter(|c| { srcs[..i] .iter() - .any(|s| column_index_ci(s.table, &c.name).is_some()) + .any(|s| s.table.column_index(&c.name).is_some()) }) .map(|c| c.name.clone()) .collect(), }; - bare_star = true; for name in names { - let right_col = column_index_ci(srcs[i].table, &name); + let right_col = srcs[i].table.column_index(&name); // The leftmost source that has the column. let left = srcs[..i] .iter() .enumerate() - .find_map(|(j, s)| column_index_ci(s.table, &name).map(|c| (j, c))); + .find_map(|(j, s)| s.table.column_index(&name).map(|c| (j, c))); let (Some(right_col), Some((left_src, left_col))) = (right_col, left) else { return Err(SqawkError::InvalidSqlQuery(format!( "Cannot join using column '{}': it must exist on both sides", @@ -597,17 +663,11 @@ impl<'a> SqlCompiler<'a> { &srcs[i].refname, &name, )); - hidden.push((i, right_col)); + hidden.insert((i, right_col)); let key = name.to_ascii_lowercase(); let left_ref = (srcs[left_src].cursor, left_col); let right_ref = (srcs[i].cursor, right_col); - match using.iter_mut().find(|u| u.refs.contains(&left_ref)) { - Some(u) => u.refs.push(right_ref), - None => using.push(UsingCol { - name: key, - refs: vec![left_ref, right_ref], - }), - } + using.merge(key, left_ref, right_ref); } } @@ -619,7 +679,6 @@ impl<'a> SqlCompiler<'a> { using, hidden, full_where: None, - bare_star, const_where: Vec::new(), outer, }; @@ -724,8 +783,16 @@ impl<'a> SqlCompiler<'a> { plan.full_where = Some(where_terms); } else { for term in where_terms { - match expr_sources(&term, &ctx) { + let sources = + expr_sources(&term, &ctx).or_else(|| self.compiled_term_sources(&term, &ctx)); + match sources { None => plan.levels[last].post_terms.push(term), + // A conjunct reading no table of this FROM (a constant, + // a test of an enclosing query's row, an uncorrelated + // subquery) is tested once, before the nest: when it is + // false no row can qualify, and in a correlated + // subquery the scan is skipped for that outer row. + Some(s) if s.is_empty() => plan.const_where.push(term), Some(s) => { let at = plan.max_level(&s); match plan.group_of(at).map(|g| g.end) { @@ -795,6 +862,16 @@ impl<'a> SqlCompiler<'a> { body: &mut dyn FnMut(&mut Self, &NameCtx<'a>) -> SqawkResult<()>, ) -> SqawkResult<()> { let ctx = plan.ctx(); + // WHERE terms that read no table are tested before anything else, as + // SQLite does: when one is false no table is opened, so a CTE or + // derived table read only here is never computed (see + // `Database::add_lazy_table`), and no hash index is built. Nothing + // is output then, not even a FULL JOIN's unmatched rows. + let skip = self.label(); + for term in &plan.const_where { + let r = self.code_expr(term, &ctx, None)?; + self.emit_jump_to(OpCode::IfZ, r, skip, 0, None, "WHERE: before the scan"); + } // A subquery's loop nest may run once per outer row. Its cursors and // hash indexes do not depend on the outer row, so they are set up // only the first time. (Not for FULL JOIN, whose per-row match marks @@ -811,50 +888,19 @@ impl<'a> SqlCompiler<'a> { "subquery cursors: open once", ); } - for (i, name) in plan.table_names.iter().enumerate() { - let cursor = plan.cursor(i); - self.emit( - OpCode::OpenRead, - cursor, - 0, - 0, - Some(name.clone()), - &format!("Open {} as cursor {}", name, cursor), - ); + // An inner level's table is opened, and its hash index built, when + // the nest first reaches that level (see `deferred_levels`); every + // other table here, before the nest. + let deferred = plan.deferred_levels(); + for (lvl, level) in plan.levels.iter().enumerate() { + if !deferred[lvl] { + self.emit_open(plan, level.src); + } } - - // Build each hash index once, before the nest: its key expressions - // read only the inner table. - for level in &plan.levels { - let Some((inner, _)) = &level.hash else { - continue; - }; - let cursor = plan.cursor(level.src); - let top = self.label(); - let done = self.label(); - self.emit_jump_to( - OpCode::Rewind, - cursor, - done, - 0, - None, - "hash build: scan inner", - ); - self.resolve(top); - let keys = self.allocate_registers(inner.len()); - for (j, e) in inner.iter().enumerate() { - self.code_expr(e, &ctx, Some(keys + j as i64))?; + for (lvl, level) in plan.levels.iter().enumerate() { + if !deferred[lvl] { + self.emit_hash_build(plan, &ctx, level)?; } - self.emit( - OpCode::HashInsert, - cursor, - keys, - inner.len() as i64, - None, - "hash build: index row", - ); - self.emit_jump_to(OpCode::Next, cursor, top, 0, None, ""); - self.resolve(done); } if once { self.resolve(opened); @@ -868,15 +914,14 @@ impl<'a> SqlCompiler<'a> { let match_labels: Vec
, - /// Names of tables that were modified (INSERT, UPDATE, DELETE, CREATE TABLE) - pub modified_tables: HashSet, /// Number of rows affected by the last DML statement (INSERT, UPDATE, DELETE) pub affected_rows: usize, /// Whether any statement in this script was a row-counting DML. @@ -78,23 +85,23 @@ pub fn execute_vm( verbose: bool, ) -> SqawkResult { if verbose { - eprintln!("VM Engine: Executing SQL via bytecode: {}", sql); + diag!("VM Engine: Executing SQL via bytecode: {}", sql); let sql_upper = sql.to_uppercase(); if sql_upper.contains("DISTINCT") { - eprintln!("Applying DISTINCT"); + diag!("Applying DISTINCT"); } if sql_upper.contains("ORDER BY") { - eprintln!("Applying ORDER BY"); + diag!("Applying ORDER BY"); } if sql_upper.contains("LIMIT") || sql_upper.contains("OFFSET") { - eprintln!("Applying LIMIT/OFFSET"); + diag!("Applying LIMIT/OFFSET"); } if sql_upper.contains("GROUP BY") { - eprintln!("Applying GROUP BY"); + diag!("Applying GROUP BY"); } if sql_upper.contains("HAVING") { - eprintln!("Applying HAVING"); + diag!("Applying HAVING"); } } @@ -113,9 +120,7 @@ pub fn execute_vm( // Executing per statement gives each its own result set and makes each // statement observe the database as the previous one left it, which is // also what `CREATE TABLE t; INSERT INTO t ...` requires. - let dialect = sqlparser::dialect::HiveDialect {}; - let statements = - sqlparser::parser::Parser::parse_sql(&dialect, sql).map_err(SqawkError::SqlParseError)?; + let statements = parse::parse_sql(sql)?; if statements.is_empty() { return Err(SqawkError::InvalidSqlQuery( @@ -123,38 +128,75 @@ pub fn execute_vm( )); } + let source = Arc::new(parse::SqlSource::new(sql)); + execute_statements( + statements, + database, + verbose, + &source, + false, + &Arc::default(), + ) +} + +/// Compile and execute parsed statements one at a time (see `execute_vm`). +/// +/// `source` is the text the statements were parsed from; it names unaliased +/// result columns. `in_derived`: the statement is a derived table's or CTE's +/// query, whose column references are named as written (see `SqlCompiler`). +/// `cte_names`: for such a query, the CTE names of the enclosing statement +/// (see `prepass::Prepared::cte_names`); empty at top level. +fn execute_statements( + statements: Vec, + database: &mut Database, + verbose: bool, + source: &Arc, + in_derived: bool, + cte_names: &Arc>, +) -> SqawkResult { let mut tables: Vec
= Vec::new(); - let mut modified_tables = HashSet::with_capacity(DEFAULT_TABLE_CAPACITY); let mut affected_rows: usize = 0; let mut dml_executed = false; - for statement in &statements { - // PHASE 0: MATERIALIZE DERIVED TABLES. + for mut statement in statements { + // BEGIN / COMMIT / ROLLBACK act on the database directly; they have + // no bytecode. + if let Some(control) = compiler_check::transaction_control(&statement)? { + match control { + compiler_check::TransactionControl::Begin => database.begin()?, + compiler_check::TransactionControl::Commit => database.commit()?, + compiler_check::TransactionControl::Rollback => database.rollback()?, + } + continue; + } + + // PHASE 0: MATERIALIZE DERIVED TABLES AND CTEs. // - // `FROM (SELECT ...) t` has no representation in the compiler, which - // only knows how to open a cursor on a named table. Rather than teach - // every FROM site about subqueries, each derived table is executed - // first and registered under its alias, and the statement is rewritten - // to reference that name. The compiler then sees an ordinary table. - let mut statement = statement.clone(); - let derived = materialize_derived_tables(&mut statement, database, verbose)?; + // `FROM (SELECT ...) t` and a CTE reference have no representation in + // the compiler, which only knows how to open a cursor on a named + // table. Each is computed first and registered under an internal + // name, and the statement is rewritten to reference that name (see + // `prepass.rs`). The compiler then sees ordinary tables. + let prepared = prepass::run(&mut statement, database, verbose, source, cte_names)?; // Phases 1-3 run inside a closure so a failure anywhere still drops - // the derived tables registered above. Leaving them behind made the - // alias unusable for the rest of the session: a failed + // the tables registered above. Leaving them behind made the alias + // unusable for the rest of the session: a failed // `SELECT bad FROM (SELECT ...) x` left `x` registered, so the next // statement using the same alias failed with "shadows an existing // table", and `.tables` listed a phantom table. let outcome = (|| -> SqawkResult<(Option
, AppliedModifications)> { // PHASE 1: SQL -> BYTECODE, against the CURRENT database state. let program = { - let mut compiler = compiler::SqlCompiler::new(database, verbose); + let mut compiler = compiler::SqlCompiler::new(database, verbose) + .with_source(Some(Arc::clone(source)), in_derived) + .with_cte_names(Arc::clone(&prepared.cte_names)); compiler.compile_statement_program(&statement)? }; if verbose { - eprintln!("Generated bytecode:"); - eprintln!("{}", program); + diag!("Generated bytecode:"); + diag!("{}", program); } // PHASE 2: EXECUTE @@ -171,14 +213,13 @@ pub fn execute_vm( Ok((result_table, applied)) })(); - // Derived tables live only for the statement that declared them, and - // must be dropped whether or not it succeeded. - for name in &derived { + // Derived tables and CTEs live only for the statement that declared + // them, and must be dropped whether or not it succeeded. + for name in &prepared.registered { database.remove_table(name); } let (result_table, applied) = outcome?; - modified_tables.extend(applied.modified_tables); // A DML statement reports its own count, INCLUDING zero. // @@ -202,272 +243,11 @@ pub fn execute_vm( Ok(VmExecutionResult { tables, - modified_tables, affected_rows, dml_executed, }) } -/// Execute every derived table in `statement` -- in the FROM of the query, -/// of a compound's arms, of subqueries in any expression, of an INSERT ... -/// SELECT or CREATE TABLE ... AS -- register each under an internal name, -/// and rewrite the FROM item to reference that name, keeping the derived -/// table's alias as the item's alias. -/// -/// Returns the names registered, so the caller can drop them once the -/// statement finishes -- a derived table is scoped to the statement that -/// declares it. -/// -/// A derived table that refers to a column of an enclosing query (inside a -/// correlated subquery) cannot be computed up front; it fails to resolve. -fn materialize_derived_tables( - statement: &mut sqlparser::ast::Statement, - database: &mut Database, - verbose: bool, -) -> SqawkResult> { - let mut registered = Vec::new(); - let result = materialize_statement(statement, database, verbose, &mut registered); - if result.is_err() { - for name in ®istered { - database.remove_table(name); - } - } - result.map(|_| registered) -} - -fn materialize_statement( - statement: &mut sqlparser::ast::Statement, - database: &mut Database, - verbose: bool, - registered: &mut Registered, -) -> SqawkResult<()> { - use sqlparser::ast::Statement; - match statement { - Statement::Query(query) => { - materialize_in_query(query, database, verbose, registered)?; - } - Statement::Insert(insert) => { - if let Some(source) = insert.source.as_mut() { - materialize_in_query(source, database, verbose, registered)?; - } - } - Statement::CreateTable(create) => { - if let Some(query) = create.query.as_mut() { - materialize_in_query(query, database, verbose, registered)?; - } - } - Statement::Update(update) => { - for a in &mut update.assignments { - materialize_in_expr(&mut a.value, database, verbose, registered)?; - } - if let Some(e) = update.selection.as_mut() { - materialize_in_expr(e, database, verbose, registered)?; - } - } - Statement::Delete(delete) => { - if let Some(e) = delete.selection.as_mut() { - materialize_in_expr(e, database, verbose, registered)?; - } - } - _ => {} - } - Ok(()) -} - -type Registered = Vec; - -fn materialize_in_query( - query: &mut sqlparser::ast::Query, - database: &mut Database, - verbose: bool, - registered: &mut Registered, -) -> SqawkResult<()> { - materialize_in_set_expr(&mut query.body, database, verbose, registered)?; - if let Some(sqlparser::ast::OrderBy { - kind: sqlparser::ast::OrderByKind::Expressions(obs), - .. - }) = query.order_by.as_mut() - { - for ob in obs { - materialize_in_expr(&mut ob.expr, database, verbose, registered)?; - } - } - Ok(()) -} - -fn materialize_in_set_expr( - body: &mut sqlparser::ast::SetExpr, - database: &mut Database, - verbose: bool, - registered: &mut Registered, -) -> SqawkResult<()> { - use sqlparser::ast::{SelectItem, SetExpr}; - match body { - SetExpr::Select(select) => { - for twj in &mut select.from { - materialize_in_factor(&mut twj.relation, database, verbose, registered)?; - for join in &mut twj.joins { - materialize_in_factor(&mut join.relation, database, verbose, registered)?; - } - } - for item in &mut select.projection { - if let SelectItem::UnnamedExpr(e) | SelectItem::ExprWithAlias { expr: e, .. } = item - { - materialize_in_expr(e, database, verbose, registered)?; - } - } - for e in select.selection.iter_mut().chain(select.having.iter_mut()) { - materialize_in_expr(e, database, verbose, registered)?; - } - Ok(()) - } - SetExpr::SetOperation { left, right, .. } => { - materialize_in_set_expr(left, database, verbose, registered)?; - materialize_in_set_expr(right, database, verbose, registered) - } - SetExpr::Query(q) => materialize_in_query(q, database, verbose, registered), - _ => Ok(()), - } -} - -/// Find the subqueries within an expression and materialize the derived -/// tables in their FROM clauses. -fn materialize_in_expr( - expr: &mut sqlparser::ast::Expr, - database: &mut Database, - verbose: bool, - registered: &mut Registered, -) -> SqawkResult<()> { - use sqlparser::ast::{Expr, FunctionArg, FunctionArgExpr, FunctionArguments}; - let mut go = |e: &mut Expr| materialize_in_expr(e, database, verbose, registered); - match expr { - Expr::Subquery(q) | Expr::Exists { subquery: q, .. } => { - materialize_in_query(q, database, verbose, registered) - } - Expr::InSubquery { expr, subquery, .. } => { - go(expr)?; - materialize_in_query(subquery, database, verbose, registered) - } - Expr::BinaryOp { left, right, .. } - | Expr::IsDistinctFrom(left, right) - | Expr::IsNotDistinctFrom(left, right) => { - go(left)?; - go(right) - } - Expr::Nested(e) - | Expr::UnaryOp { expr: e, .. } - | Expr::Cast { expr: e, .. } - | Expr::IsNull(e) - | Expr::IsNotNull(e) => go(e), - Expr::Between { - expr, low, high, .. - } => { - go(expr)?; - go(low)?; - go(high) - } - Expr::InList { expr, list, .. } => { - go(expr)?; - list.iter_mut().try_for_each(go) - } - Expr::Like { expr, pattern, .. } | Expr::ILike { expr, pattern, .. } => { - go(expr)?; - go(pattern) - } - Expr::Case { - operand, - conditions, - else_result, - .. - } => { - if let Some(o) = operand { - go(o)?; - } - for w in conditions { - go(&mut w.condition)?; - go(&mut w.result)?; - } - if let Some(e) = else_result { - go(e)?; - } - Ok(()) - } - Expr::Function(f) => { - if let FunctionArguments::List(list) = &mut f.args { - for arg in &mut list.args { - if let FunctionArg::Unnamed(FunctionArgExpr::Expr(e)) = arg { - go(e)?; - } - } - } - Ok(()) - } - _ => Ok(()), - } -} - -fn materialize_in_factor( - factor: &mut sqlparser::ast::TableFactor, - database: &mut Database, - verbose: bool, - registered: &mut Registered, -) -> SqawkResult<()> { - let (subquery, alias) = match factor { - sqlparser::ast::TableFactor::Derived { lateral: true, .. } => { - return Err(SqawkError::UnsupportedSqlFeature( - "LATERAL subqueries are not supported".into(), - )) - } - sqlparser::ast::TableFactor::Derived { - subquery, alias, .. - } => (subquery.clone(), alias.clone()), - _ => return Ok(()), - }; - if alias.as_ref().is_some_and(|a| !a.columns.is_empty()) { - return Err(SqawkError::UnsupportedSqlFeature( - "Column aliases on a FROM item are not supported".into(), - )); - } - - // Nested derived tables are materialized innermost-first. - let mut inner = *subquery; - materialize_in_query(&mut inner, database, verbose, registered)?; - - // Registered under an internal name, so that an alias may repeat - // across subqueries and may shadow a real table, as in SQL. (The name - // must survive being rendered back to SQL for an enclosing derived - // table, so it is a plain identifier.) - let name = (1..) - .map(|i| format!("sqawk_derived_{}", i)) - .find(|n| !database.has_table(n)) - .expect("an unused name"); - - let inner_sql = inner.to_string(); - let result = execute_vm(&inner_sql, database, verbose)?; - let mut table = result.tables.into_iter().next_back().ok_or_else(|| { - SqawkError::InvalidSqlQuery("Subquery in FROM produced no result".to_string()) - })?; - table.set_name(name.clone()); - database.add_table(name.clone(), table)?; - registered.push(name.clone()); - - *factor = sqlparser::ast::TableFactor::Table { - name: sqlparser::ast::ObjectName(vec![sqlparser::ast::ObjectNamePart::Identifier( - sqlparser::ast::Ident::new(name), - )]), - alias, - args: None, - with_hints: Vec::new(), - version: None, - partitions: Vec::new(), - with_ordinality: false, - json_path: None, - sample: None, - index_hints: Vec::new(), - }; - Ok(()) -} - /// Whether a statement's affected-row count is meaningful to report. /// /// The INSERT/UPDATE/DELETE family, matching what SQL's `changes()` covers. A @@ -491,13 +271,14 @@ fn is_row_counting_dml(statement: &sqlparser::ast::Statement) -> bool { ) } -/// Tables touched and rows affected by one statement's modifications. +/// Rows affected by one statement's modifications. struct AppliedModifications { - modified_tables: HashSet, affected_rows: usize, } -/// Apply a statement's pending modifications to the database. +/// Apply a statement's pending modifications to the database, marking each +/// table they change as modified (DROP TABLE takes the mark away with the +/// table). fn apply_modifications( database: &mut Database, modifications: Vec, @@ -505,9 +286,6 @@ fn apply_modifications( ) -> SqawkResult { // PHASE 3: APPLY MODIFICATIONS TO DATABASE - // Track which tables were modified - let mut modified_tables = HashSet::with_capacity(DEFAULT_TABLE_CAPACITY); - // Track affected rows count for DML operations let mut affected_rows: usize = 0; @@ -532,7 +310,7 @@ fn apply_modifications( *insert_counts_by_table .entry(table_name.clone()) .or_insert(0) += 1; - modified_tables.insert(table_name); + database.mark_modified(&table_name); } engine::TableModification::Replace { table_name, @@ -545,7 +323,7 @@ fn apply_modifications( *replace_counts_by_table .entry(table_name.clone()) .or_insert(0) += 1; - modified_tables.insert(table_name); + database.mark_modified(&table_name); } engine::TableModification::Delete { table_name, @@ -562,40 +340,58 @@ fn apply_modifications( columns, file_path, delimiter, + header, + comments, } => { - // Convert engine ColumnDef to table ColumnDefinition + // Each column keeps its declared type name; its affinity + // follows SQLite's rules. let schema: Vec = columns .into_iter() .map(|col| crate::table::ColumnDefinition { name: col.name, - data_type: match col.data_type.as_str() { - "INTEGER" => crate::table::DataType::Integer, - "REAL" => crate::table::DataType::Float, - "BOOLEAN" => crate::table::DataType::Boolean, - _ => crate::table::DataType::Text, - }, + data_type: crate::table::DataType::from_declared(&col.data_type), + decl_type: Some(col.data_type), }) .collect(); - // Create the table with schema - let file_path_buf = file_path.map(std::path::PathBuf::from); - let table = crate::table::Table::new_with_schema( + let Some(path) = file_path else { + // No LOCATION: a scratch table, never written. + let table = + crate::table::Table::new_with_schema(&table_name, schema, None, delimiter); + database.add_table(table_name.clone(), table)?; + database.mark_modified(&table_name); + continue; + }; + // The declared columns are the schema of the LOCATION file: + // an existing file is loaded under them (and is not modified + // by loading); a new one is created on --write. + if database.has_table(&table_name) { + return Err(crate::error::SqawkError::TableAlreadyExists(table_name)); + } + let options = crate::file_handler::LocationOptions { + delimiter, + header, + comments, + }; + let (table, existed) = crate::file_handler::load_location( + database, &table_name, + &path, schema, - file_path_buf, - delimiter, - ); - - // Add to database + &options, + )?; database.add_table(table_name.clone(), table)?; - modified_tables.insert(table_name); + if !existed { + database.mark_modified(&table_name); + } } engine::TableModification::DropTable { table_name } => { // Remove table from database if !database.remove_table(&table_name) { return Err(crate::error::SqawkError::TableNotFound(table_name)); } - // Note: we don't add to modified_tables since the table is gone + // remove_table also drops the table from the modified set: + // a dropped table is not written back. } engine::TableModification::AlterTableAddColumn { table_name, @@ -603,21 +399,19 @@ fn apply_modifications( column_type, } => { let table = database.get_table_mut(&table_name)?; - let data_type = match column_type.as_str() { - "INTEGER" => crate::table::DataType::Integer, - "REAL" => crate::table::DataType::Float, - "BOOLEAN" => crate::table::DataType::Boolean, - _ => crate::table::DataType::Text, - }; - table.add_column_with_default(column_name, data_type, crate::table::Value::Null)?; - modified_tables.insert(table_name); + table.add_column_with_default( + column_name, + column_type, + crate::table::Value::Null, + )?; + database.mark_modified(&table_name); } engine::TableModification::Truncate { table_name } => { let table = database.get_table_mut(&table_name)?; let row_count = table.row_count(); table.clear_rows()?; affected_rows += row_count; - modified_tables.insert(table_name); + database.mark_modified(&table_name); } } } @@ -642,7 +436,7 @@ fn apply_modifications( // Keep the rows not being deleted (and their comment lines in place) table.retain_rows(|idx| !indices_to_delete.contains(&idx)); - modified_tables.insert(table_name.clone()); + database.mark_modified(&table_name); // If we have equal deletes and inserts for this table, it's an UPDATE // Otherwise it's a DELETE @@ -654,13 +448,13 @@ fn apply_modifications( // UPDATE: count the number of rows updated affected_rows += delete_count; if verbose { - eprintln!("Updated {} rows", delete_count); + diag!("Updated {} rows", delete_count); } } else { // DELETE: count the number of rows deleted affected_rows += delete_count; if verbose { - eprintln!("Deleted {} rows", delete_count); + diag!("Deleted {} rows", delete_count); } } } @@ -668,12 +462,9 @@ fn apply_modifications( for (_table, count) in replace_counts_by_table { affected_rows += count; if verbose { - eprintln!("Updated {} rows", count); + diag!("Updated {} rows", count); } } - Ok(AppliedModifications { - modified_tables, - affected_rows, - }) + Ok(AppliedModifications { affected_rows }) } diff --git a/src/vm/parse.rs b/src/vm/parse.rs new file mode 100644 index 0000000..47c7b0d --- /dev/null +++ b/src/vm/parse.rs @@ -0,0 +1,1176 @@ +//! The one place SQL text becomes an AST. +//! +//! [`parse_sql`] is used by every statement sqawk runs. Beyond calling +//! sqlparser with [`SqawkDialect`] (SQLite operator precedence) it: +//! +//! - rewrites every compound SELECT left-deep in source order, because +//! SQLite gives `UNION`, `UNION ALL`, `INTERSECT` and `EXCEPT` equal +//! precedence, left to right, where sqlparser binds `INTERSECT` tighter +//! (`A UNION ALL B INTERSECT C` is `(A UNION ALL B) INTERSECT C`). A +//! parenthesized operand (`SetExpr::Query`) stays one operand; +//! - enforces SQLite's limits: an expression tree at most +//! [`MAX_EXPR_DEPTH`] deep and a compound SELECT of at most +//! [`MAX_COMPOUND_TERMS`] terms. +//! +//! The parser builds a chain such as `1+1+...+1` in a loop, so its depth is +//! unbounded, while everything downstream (compiler, `Clone`, `Display`) +//! recurses once per level. The limit walk therefore uses an explicit stack +//! and runs before anything else touches the tree. + +use sqlparser::ast::{ + Expr, FromTable, FunctionArg, FunctionArgExpr, FunctionArgumentClause, FunctionArguments, + GroupByExpr, JoinConstraint, JoinOperator, LimitClause, NamedWindowExpr, OrderByExpr, + OrderByKind, Query, Select, SelectItem, SetExpr, SetOperator, SetQuantifier, Statement, + TableFactor, TableWithJoins, Values, WindowFrameBound, WindowSpec, WindowType, +}; +use sqlparser::keywords::Keyword; +use sqlparser::parser::Parser; +use sqlparser::tokenizer::{Location, Token, TokenWithSpan, Tokenizer}; + +use super::dialect::SqawkDialect; +use crate::error::{SqawkError, SqawkResult}; + +/// SQLite's `SQLITE_MAX_EXPR_DEPTH`. +pub const MAX_EXPR_DEPTH: usize = 1000; +/// SQLite's `SQLITE_MAX_COMPOUND_SELECT`. +pub const MAX_COMPOUND_TERMS: usize = 500; + +/// Parse `sql` into statements: SQLite precedence, hex integer literals, +/// compound SELECTs left-deep, depth and compound-term limits enforced. +pub fn parse_sql(sql: &str) -> SqawkResult> { + let mut tokens = Tokenizer::new(&SqawkDialect, sql) + .tokenize_with_location() + .map_err(|e| SqawkError::SqlParseError(e.into()))?; + rewrite_hex_literals(sql, &mut tokens)?; + drop_materialized_hints(&mut tokens); + let mut statements = Parser::new(&SqawkDialect) + .with_tokens_with_locations(tokens) + .parse_statements() + .map_err(SqawkError::SqlParseError)?; + for statement in &mut statements { + normalize(statement)?; + } + Ok(statements) +} + +/// The byte range in `sql` of each token, from the tokens' (line, column) +/// locations. +fn token_offsets(sql: &str, tokens: &[TokenWithSpan]) -> Vec<(usize, usize)> { + // Locations only grow, so one forward walk maps them all. + let mut walk = (1u64, 1u64, sql.char_indices().peekable()); + let mut offset = |loc: Location| -> usize { + let (line, col, chars) = &mut walk; + while (*line, *col) < (loc.line, loc.column) { + match chars.next() { + Some((_, '\n')) => { + *line += 1; + *col = 1; + } + Some(_) => *col += 1, + None => break, + } + } + chars.peek().map_or(sql.len(), |(i, _)| *i) + }; + tokens + .iter() + .map(|t| { + let start = offset(t.span.start); + (start, offset(t.span.end)) + }) + .collect() +} + +/// SQLite's hexadecimal integer literals, `0x1F` / `0X1F`: up to 16 +/// significant hex digits, read as a 64-bit two's-complement value +/// (`0xFFFFFFFFFFFFFFFF` is -1). Each becomes a decimal number token. +/// +/// The tokenizer reads `0x1F` as a blob literal (the token `X'1F'` makes) +/// and `0X1F` as a word, so both are found here and told apart from a real +/// `X'1F'` by their source text. As in SQLite the literal ends at the last +/// hex digit, so `0x1g` is `0x1 AS g`; more than 16 significant digits are +/// the error "hex literal too big", and a `0x` with no hex digit after it +/// is an unrecognized token. +/// +/// The tokenizer also reads `~~` as one (PostgreSQL) operator; SQLite reads +/// two `~`, so it is split here. +fn rewrite_hex_literals(sql: &str, tokens: &mut Vec) -> SqawkResult<()> { + let hex_word = |t: &Token| { + matches!(t, Token::Word(w) if w.quote_style.is_none() + && (w.value.starts_with("0x") || w.value.starts_with("0X"))) + }; + if !tokens.iter().any(|t| { + matches!(t.token, Token::HexStringLiteral(_) | Token::DoubleTilde) || hex_word(&t.token) + }) { + return Ok(()); + } + let offsets = token_offsets(sql, tokens); + let mut out = Vec::with_capacity(tokens.len()); + for (k, t) in std::mem::take(tokens).into_iter().enumerate() { + let (start, end) = offsets[k]; + let text = sql.get(start..end).unwrap_or_default(); + let is_hex = match &t.token { + Token::HexStringLiteral(_) => text.starts_with("0x") || text.starts_with("0X"), + Token::DoubleTilde => { + let mut second = t.clone(); + second.span.start.column += 1; + let mut first = t; + first.token = Token::Tilde; + first.span.end.column = first.span.start.column + 1; + second.token = Token::Tilde; + out.push(first); + out.push(second); + continue; + } + other => hex_word(other), + }; + if !is_hex { + out.push(t); + continue; + } + let after = &text[2..]; + let n = after.bytes().take_while(u8::is_ascii_hexdigit).count(); + if n == 0 { + // `0x` alone, or `0xZZ` (the tokenizer splits that in two). + let glued_end = match tokens_after(&offsets, k) { + Some((s, e)) if s == end => e, + _ => end, + }; + return Err(SqawkError::InvalidSqlQuery(format!( + "unrecognized token: \"{}\"", + sql.get(start..glued_end).unwrap_or(text) + ))); + } + let (digits, rest) = after.split_at(n); + let significant = digits.trim_start_matches('0'); + if significant.len() > 16 { + return Err(SqawkError::InvalidSqlQuery(format!( + "hex literal too big: {}{}", + &text[..2], + digits + ))); + } + let value = u64::from_str_radix(significant, 16).unwrap_or(0) as i64; + let mut number = t.clone(); + number.token = Token::Number(value.to_string(), false); + if !rest.is_empty() { + number.span.end.column = number.span.start.column + 2 + n as u64; + } + out.push(number); + if !rest.is_empty() { + // `0X1G`, one word to the tokenizer: the rest is a word of its own. + let mut word = t; + word.span.start.column += 2 + n as u64; + word.token = Token::make_word(rest, None); + out.push(word); + } + } + *tokens = out; + Ok(()) +} + +/// SQLite's CTE hints `name AS MATERIALIZED (...)` and +/// `name AS NOT MATERIALIZED (...)` only choose a query plan, and the +/// dialect does not parse them, so they are dropped from the token stream. +/// +/// Only an `AS` directly inside a WITH list counts: from `WITH` until the +/// list ends at its statement keyword (SELECT, VALUES, INSERT, UPDATE, +/// DELETE), at `;` or at the parenthesis enclosing the WITH, at the WITH's +/// own parenthesis depth. `CAST(x AS materialized)` is untouched. +fn drop_materialized_hints(tokens: &mut Vec) { + let is_kw = |t: &Token, kw: Keyword| matches!(t, Token::Word(w) if w.keyword == kw && w.quote_style.is_none()); + if !tokens + .iter() + .any(|t| is_kw(&t.token, Keyword::MATERIALIZED)) + { + return; + } + let significant: Vec = (0..tokens.len()) + .filter(|&i| !matches!(tokens[i].token, Token::Whitespace(_))) + .collect(); + let mut drop = vec![false; tokens.len()]; + // The parenthesis depth of each WITH list being read. + let mut lists: Vec = Vec::new(); + let mut depth = 0usize; + for (k, &i) in significant.iter().enumerate() { + let t = &tokens[i].token; + match t { + Token::LParen => depth += 1, + Token::RParen => { + if lists.last() == Some(&depth) { + lists.pop(); + } + depth = depth.saturating_sub(1); + } + Token::SemiColon => lists.clear(), + _ if is_kw(t, Keyword::WITH) => lists.push(depth), + _ if lists.last() == Some(&depth) => { + let ends = [ + Keyword::SELECT, + Keyword::VALUES, + Keyword::INSERT, + Keyword::UPDATE, + Keyword::DELETE, + ] + .iter() + .any(|&kw| is_kw(t, kw)); + if ends { + lists.pop(); + } else if is_kw(t, Keyword::AS) { + let next = |n: usize| significant.get(k + n).map(|&j| &tokens[j].token); + let skip = match (next(1), next(2), next(3)) { + (Some(m), Some(Token::LParen), _) if is_kw(m, Keyword::MATERIALIZED) => 1, + (Some(n), Some(m), Some(Token::LParen)) + if is_kw(n, Keyword::NOT) && is_kw(m, Keyword::MATERIALIZED) => + { + 2 + } + _ => 0, + }; + for n in 1..=skip { + drop[significant[k + n]] = true; + } + } + } + _ => {} + } + } + let mut i = 0; + tokens.retain(|_| { + i += 1; + !drop[i - 1] + }); +} + +/// The byte range of the token after the `k`-th, if any. +fn tokens_after(offsets: &[(usize, usize)], k: usize) -> Option<(usize, usize)> { + offsets.get(k + 1).copied() +} + +fn too_deep() -> SqawkError { + SqawkError::InvalidSqlQuery(format!( + "expression tree is too large (maximum depth {})", + MAX_EXPR_DEPTH + )) +} + +/// One node of a statement's tree: the unit of the walks that must reach +/// every query, FROM item and expression of a statement -- the limit and +/// compound walk here, and the derived-table / CTE pre-pass (`prepass.rs`). +/// Both enumerate children through [`for_each_child`], so they reach the +/// same places. +pub(crate) enum Node<'a> { + Statement(&'a mut Statement), + Query(&'a mut Query), + SetExpr(&'a mut SetExpr), + Select(&'a mut Select), + Table(&'a mut TableWithJoins), + Factor(&'a mut TableFactor), + Expr(&'a mut Expr), +} + +/// Rewrite compounds and check limits over one statement, iteratively. +/// +/// Each stack entry carries a depth: for an `Expr` its depth (1 for the root +/// of an expression), for the other kinds the depth of the expression that +/// contains them (0 at statement level). After `rewrite_compound` a compound +/// is left-deep, so walking it with the stack (not recursion) keeps 500 +/// terms cheap. +fn normalize(statement: &mut Statement) -> SqawkResult<()> { + let mut stack: Vec<(Node, usize)> = vec![(Node::Statement(statement), 0)]; + while let Some((node, d)) = stack.pop() { + match node { + Node::Expr(e) => { + if d > MAX_EXPR_DEPTH { + return Err(too_deep()); + } + strip_main_schema(e)?; + // Parentheses are not a node in SQLite's tree: `-(x)` is as + // deep as `-x`. + let child = if matches!(e, Expr::Nested(_)) { + d + } else { + d + 1 + }; + for_each_child(Node::Expr(e), &mut |c| stack.push((c, child))); + } + Node::Query(q) => { + rewrite_compound(&mut q.body)?; + push_children(Node::Query(q), d, &mut stack); + } + other => push_children(other, d, &mut stack), + } + } + Ok(()) +} + +/// Push the children of a node that is not an expression: an expression +/// child starts one level below `d`, anything else stays at `d`. +fn push_children<'a>(node: Node<'a>, d: usize, stack: &mut Vec<(Node<'a>, usize)>) { + for_each_child(node, &mut |c| { + let depth = if matches!(c, Node::Expr(_)) { d + 1 } else { d }; + stack.push((c, depth)); + }); +} + +/// Call `f` on each child of `node`: the queries, FROM items and +/// expressions it contains, including a query's CTE bodies and the +/// statement inside `WITH ... INSERT / UPDATE / DELETE`. FROM items come +/// before the expressions of a SELECT, so a walk resolves them first. +/// +/// Expressions are enumerated exhaustively (a new `Expr` variant fails to +/// compile in [`expr_children`]); query clauses sqawk rejects (TOP, SORT BY, +/// LATERAL VIEW, ...) are not visited. +pub(crate) fn for_each_child<'a>(node: Node<'a>, f: &mut dyn FnMut(Node<'a>)) { + match node { + Node::Statement(s) => statement_children(s, f), + Node::Query(q) => { + if let Some(with) = q.with.as_mut() { + for cte in &mut with.cte_tables { + f(Node::Query(&mut cte.query)); + } + } + f(Node::SetExpr(&mut q.body)); + if let Some(ob) = q.order_by.as_mut() { + if let OrderByKind::Expressions(exprs) = &mut ob.kind { + order_by_children(exprs, f); + } + } + match q.limit_clause.as_mut() { + Some(LimitClause::LimitOffset { + limit, + offset, + limit_by, + }) => { + if let Some(l) = limit { + f(Node::Expr(l)); + } + if let Some(o) = offset { + f(Node::Expr(&mut o.value)); + } + for e in limit_by { + f(Node::Expr(e)); + } + } + Some(LimitClause::OffsetCommaLimit { offset, limit }) => { + f(Node::Expr(offset)); + f(Node::Expr(limit)); + } + None => {} + } + } + Node::SetExpr(s) => match s { + SetExpr::Select(sel) => f(Node::Select(sel)), + SetExpr::Query(q) => f(Node::Query(q)), + SetExpr::SetOperation { left, right, .. } => { + f(Node::SetExpr(left)); + f(Node::SetExpr(right)); + } + SetExpr::Values(Values { rows, .. }) => { + for row in rows { + for e in &mut row.content { + f(Node::Expr(e)); + } + } + } + SetExpr::Insert(stmt) | SetExpr::Update(stmt) | SetExpr::Delete(stmt) => { + f(Node::Statement(stmt)) + } + SetExpr::Table(_) | SetExpr::Merge(_) => {} + }, + Node::Select(sel) => { + for t in &mut sel.from { + f(Node::Table(t)); + } + for item in &mut sel.projection { + match item { + SelectItem::UnnamedExpr(e) + | SelectItem::ExprWithAlias { expr: e, .. } + | SelectItem::ExprWithAliases { expr: e, .. } => f(Node::Expr(e)), + SelectItem::QualifiedWildcard(..) | SelectItem::Wildcard(_) => {} + } + } + if let Some(e) = sel.selection.as_mut() { + f(Node::Expr(e)); + } + if let GroupByExpr::Expressions(exprs, _) = &mut sel.group_by { + for e in exprs { + f(Node::Expr(e)); + } + } + if let Some(e) = sel.having.as_mut() { + f(Node::Expr(e)); + } + if let Some(sqlparser::ast::Distinct::On(exprs)) = sel.distinct.as_mut() { + for e in exprs { + f(Node::Expr(e)); + } + } + for w in &mut sel.named_window { + if let NamedWindowExpr::WindowSpec(spec) = &mut w.1 { + window_spec_children(spec, f); + } + } + if let Some(e) = sel.qualify.as_mut() { + f(Node::Expr(e)); + } + } + Node::Table(t) => { + f(Node::Factor(&mut t.relation)); + for join in &mut t.joins { + f(Node::Factor(&mut join.relation)); + if let Some(JoinConstraint::On(e)) = join_constraint(&mut join.join_operator) { + f(Node::Expr(e)); + } + } + } + Node::Factor(fac) => match fac { + TableFactor::Derived { subquery, .. } => f(Node::Query(subquery)), + TableFactor::NestedJoin { + table_with_joins, .. + } => f(Node::Table(table_with_joins)), + _ => {} + }, + Node::Expr(e) => expr_children(e, f), + } +} + +/// The children of a statement: its query, its FROM items and expressions. +fn statement_children<'a>(statement: &'a mut Statement, f: &mut dyn FnMut(Node<'a>)) { + match statement { + Statement::Query(q) => f(Node::Query(q)), + Statement::Insert(insert) => { + if let Some(source) = insert.source.as_deref_mut() { + f(Node::Query(source)); + } + } + Statement::Update(update) => { + f(Node::Table(&mut update.table)); + if let Some( + sqlparser::ast::UpdateTableFromKind::BeforeSet(from) + | sqlparser::ast::UpdateTableFromKind::AfterSet(from), + ) = update.from.as_mut() + { + for t in from { + f(Node::Table(t)); + } + } + for a in &mut update.assignments { + f(Node::Expr(&mut a.value)); + } + if let Some(e) = update.selection.as_mut() { + f(Node::Expr(e)); + } + } + Statement::Delete(delete) => { + let (FromTable::WithFromKeyword(from) | FromTable::WithoutKeyword(from)) = + &mut delete.from; + for t in from { + f(Node::Table(t)); + } + for t in delete.using.iter_mut().flatten() { + f(Node::Table(t)); + } + if let Some(e) = delete.selection.as_mut() { + f(Node::Expr(e)); + } + } + Statement::CreateTable(create) => { + if let Some(q) = create.query.as_deref_mut() { + f(Node::Query(q)); + } + } + _ => {} + } +} + +pub(crate) fn join_constraint(op: &mut JoinOperator) -> Option<&mut JoinConstraint> { + match op { + JoinOperator::Join(c) + | JoinOperator::Inner(c) + | JoinOperator::Left(c) + | JoinOperator::LeftOuter(c) + | JoinOperator::Right(c) + | JoinOperator::RightOuter(c) + | JoinOperator::FullOuter(c) + | JoinOperator::CrossJoin(c) + | JoinOperator::Semi(c) + | JoinOperator::LeftSemi(c) + | JoinOperator::RightSemi(c) + | JoinOperator::Anti(c) + | JoinOperator::LeftAnti(c) + | JoinOperator::RightAnti(c) + | JoinOperator::StraightJoin(c) + | JoinOperator::AsOf { constraint: c, .. } => Some(c), + JoinOperator::CrossApply + | JoinOperator::OuterApply + | JoinOperator::ArrayJoin + | JoinOperator::LeftArrayJoin + | JoinOperator::InnerArrayJoin => None, + } +} + +fn order_by_children<'a>(exprs: &'a mut [OrderByExpr], f: &mut dyn FnMut(Node<'a>)) { + for ob in exprs { + f(Node::Expr(&mut ob.expr)); + } +} + +fn window_spec_children<'a>(spec: &'a mut WindowSpec, f: &mut dyn FnMut(Node<'a>)) { + for e in &mut spec.partition_by { + f(Node::Expr(e)); + } + order_by_children(&mut spec.order_by, f); + if let Some(frame) = spec.window_frame.as_mut() { + for bound in std::iter::once(&mut frame.start_bound).chain(frame.end_bound.as_mut()) { + if let WindowFrameBound::Preceding(Some(e)) | WindowFrameBound::Following(Some(e)) = + bound + { + f(Node::Expr(e)); + } + } + } +} + +fn function_arg_children<'a>(args: &'a mut FunctionArguments, f: &mut dyn FnMut(Node<'a>)) { + match args { + FunctionArguments::None => {} + FunctionArguments::Subquery(q) => f(Node::Query(q)), + FunctionArguments::List(list) => { + for arg in &mut list.args { + let (FunctionArg::Named { arg, .. } + | FunctionArg::ExprNamed { arg, .. } + | FunctionArg::Unnamed(arg)) = arg; + if let FunctionArgExpr::Expr(e) = arg { + f(Node::Expr(e)); + } + } + for clause in &mut list.clauses { + match clause { + FunctionArgumentClause::OrderBy(obs) => order_by_children(obs, f), + FunctionArgumentClause::Limit(e) => f(Node::Expr(e)), + _ => {} + } + } + } + } +} + +/// `main.t.c` is `t.c`: every table lives in SQLite's `main` schema. Any +/// other three-part name, or a longer one, names no column. +fn strip_main_schema(e: &mut Expr) -> SqawkResult<()> { + let Expr::CompoundIdentifier(parts) = e else { + return Ok(()); + }; + if parts.len() <= 2 { + return Ok(()); + } + if parts.len() == 3 && parts[0].value.eq_ignore_ascii_case("main") { + parts.remove(0); + return Ok(()); + } + let name: Vec<&str> = parts.iter().map(|p| p.value.as_str()).collect(); + Err(SqawkError::ColumnNotFound(name.join("."))) +} + +/// The children of an expression: its operand expressions and subqueries. +fn expr_children<'a>(e: &'a mut Expr, stack: &mut dyn FnMut(Node<'a>)) { + let push = |stack: &mut dyn FnMut(Node<'a>), e: &'a mut Expr| stack(Node::Expr(e)); + match e { + Expr::Nested(a) + | Expr::IsFalse(a) + | Expr::IsNotFalse(a) + | Expr::IsTrue(a) + | Expr::IsNotTrue(a) + | Expr::IsNull(a) + | Expr::IsNotNull(a) + | Expr::IsUnknown(a) + | Expr::IsNotUnknown(a) + | Expr::IsNormalized { expr: a, .. } + | Expr::UnaryOp { expr: a, .. } + | Expr::Cast { expr: a, .. } + | Expr::Extract { expr: a, .. } + | Expr::Ceil { expr: a, .. } + | Expr::Floor { expr: a, .. } + | Expr::Collate { expr: a, .. } + | Expr::Prefixed { value: a, .. } + | Expr::Named { expr: a, .. } + | Expr::OuterJoin(a) + | Expr::Prior(a) + | Expr::CompoundFieldAccess { root: a, .. } + | Expr::JsonAccess { value: a, .. } => push(stack, a), + Expr::IsDistinctFrom(a, b) + | Expr::IsNotDistinctFrom(a, b) + | Expr::BinaryOp { + left: a, right: b, .. + } + | Expr::Like { + expr: a, + pattern: b, + .. + } + | Expr::ILike { + expr: a, + pattern: b, + .. + } + | Expr::SimilarTo { + expr: a, + pattern: b, + .. + } + | Expr::RLike { + expr: a, + pattern: b, + .. + } + | Expr::AnyOp { + left: a, right: b, .. + } + | Expr::AllOp { + left: a, right: b, .. + } + | Expr::InUnnest { + expr: a, + array_expr: b, + .. + } + | Expr::AtTimeZone { + timestamp: a, + time_zone: b, + } + | Expr::Position { expr: a, r#in: b } => { + push(stack, a); + push(stack, b); + } + Expr::InList { expr, list, .. } => { + push(stack, expr); + for item in list { + push(stack, item); + } + } + Expr::InSubquery { expr, subquery, .. } => { + push(stack, expr); + stack(Node::Query(subquery)); + } + Expr::Between { + expr, low, high, .. + } => { + push(stack, expr); + push(stack, low); + push(stack, high); + } + Expr::Convert { expr, styles, .. } => { + push(stack, expr); + for s in styles { + push(stack, s); + } + } + Expr::Substring { + expr, + substring_from, + substring_for, + .. + } => { + push(stack, expr); + for x in [substring_from, substring_for].into_iter().flatten() { + push(stack, x); + } + } + Expr::Trim { + expr, + trim_what, + trim_characters, + .. + } => { + push(stack, expr); + if let Some(w) = trim_what { + push(stack, w); + } + for c in trim_characters.iter_mut().flatten() { + push(stack, c); + } + } + Expr::Overlay { + expr, + overlay_what, + overlay_from, + overlay_for, + } => { + push(stack, expr); + push(stack, overlay_what); + push(stack, overlay_from); + if let Some(f) = overlay_for { + push(stack, f); + } + } + Expr::Function(f) => { + function_arg_children(&mut f.parameters, stack); + function_arg_children(&mut f.args, stack); + if let Some(filter) = f.filter.as_mut() { + push(stack, filter); + } + if let Some(WindowType::WindowSpec(spec)) = f.over.as_mut() { + window_spec_children(spec, stack); + } + order_by_children(&mut f.within_group, stack); + } + Expr::Case { + operand, + conditions, + else_result, + .. + } => { + if let Some(o) = operand { + push(stack, o); + } + for cw in conditions { + push(stack, &mut cw.condition); + push(stack, &mut cw.result); + } + if let Some(e) = else_result { + push(stack, e); + } + } + Expr::Exists { subquery, .. } | Expr::Subquery(subquery) => stack(Node::Query(subquery)), + Expr::GroupingSets(sets) | Expr::Cube(sets) | Expr::Rollup(sets) => { + for e in sets.iter_mut().flatten() { + push(stack, e); + } + } + Expr::Tuple(items) | Expr::Struct { values: items, .. } => { + for e in items { + push(stack, e); + } + } + Expr::Array(arr) => { + for e in &mut arr.elem { + push(stack, e); + } + } + Expr::Dictionary(fields) => { + for f in fields { + push(stack, &mut f.value); + } + } + Expr::Map(map) => { + for entry in &mut map.entries { + push(stack, &mut entry.key); + push(stack, &mut entry.value); + } + } + Expr::Interval(i) => push(stack, &mut i.value), + Expr::Lambda(l) => push(stack, &mut l.body), + Expr::MemberOf(m) => { + push(stack, &mut m.value); + push(stack, &mut m.array); + } + Expr::Identifier(_) + | Expr::CompoundIdentifier(_) + | Expr::Value(_) + | Expr::TypedString(_) + | Expr::MatchAgainst { .. } + | Expr::Wildcard(_) + | Expr::QualifiedWildcard(..) => {} + } +} + +/// Rebuild a compound SELECT left-deep in source order and count its terms. +/// +/// sqlparser nests a tighter-binding `INTERSECT` as a right operand; an +/// in-order walk of the `SetOperation` tree recovers the source sequence +/// `t0 op1 t1 op2 t2 ...`, which SQLite evaluates strictly left to right. +fn rewrite_compound(body: &mut Box) -> SqawkResult<()> { + if !matches!(**body, SetExpr::SetOperation { .. }) { + return Ok(()); + } + let placeholder = SetExpr::Values(Values { + explicit_row: false, + value_keyword: false, + rows: Vec::new(), + }); + let root = std::mem::replace(body, Box::new(placeholder)); + + enum Work { + Node(Box), + Op(SetOperator, SetQuantifier), + } + let mut terms: Vec> = Vec::new(); + let mut ops: Vec<(SetOperator, SetQuantifier)> = Vec::new(); + let mut work = vec![Work::Node(root)]; + while let Some(w) = work.pop() { + match w { + Work::Node(node) => match *node { + SetExpr::SetOperation { + left, + op, + set_quantifier, + right, + } => { + work.push(Work::Node(right)); + work.push(Work::Op(op, set_quantifier)); + work.push(Work::Node(left)); + } + term => terms.push(Box::new(term)), + }, + Work::Op(op, q) => ops.push((op, q)), + } + if terms.len() > MAX_COMPOUND_TERMS { + return Err(SqawkError::InvalidSqlQuery( + "too many terms in compound SELECT".to_string(), + )); + } + } + + let mut terms = terms.into_iter(); + let mut acc = terms.next().expect("a compound has at least two terms"); + for ((op, set_quantifier), right) in ops.into_iter().zip(terms) { + acc = Box::new(SetExpr::SetOperation { + left: acc, + op, + set_quantifier, + right, + }); + } + *body = acc; + Ok(()) +} + +/// The text a statement was parsed from, tokenized with byte offsets, so a +/// result column can be named by its expression exactly as written. +/// +/// sqlparser's expression spans do not cover a whole expression (`-x` +/// spans `x`, `CAST(x AS INT)` spans `x`, `(a)` spans `a`), so names are +/// cut from the token stream instead: the tokens between a SELECT keyword +/// and the end of its result list, split at top-level commas. +pub struct SqlSource { + text: String, + /// Every token in source order: its (line, column) location and its + /// byte range in `text`. + tokens: Vec<(Token, (u64, u64), usize, usize)>, +} + +impl SqlSource { + /// Tokenize `sql` as [`parse_sql`] does. A text that does not tokenize + /// yields a source without tokens, which names nothing. + pub fn new(sql: &str) -> Self { + let mut tokens = Vec::new(); + if let Ok(toks) = Tokenizer::new(&SqawkDialect, sql).tokenize_with_location() { + let offsets = token_offsets(sql, &toks); + for (t, (start, end)) in toks.into_iter().zip(offsets) { + tokens.push(( + t.token, + (t.span.start.line, t.span.start.column), + start, + end, + )); + } + } + SqlSource { + text: sql.to_string(), + tokens, + } + } + + /// The source text of each item of `select`'s result list, from its + /// first token to its last (an item's alias included), or `None` when + /// the list cannot be located in the text. + pub fn select_item_texts(&self, select: &Select) -> Option> { + let start = select.select_token.0.span.start; + if start.line == 0 { + return None; + } + // The SELECT keyword's token: the one starting at that location. + let mut i = self + .tokens + .binary_search_by_key(&(start.line, start.column), |t| t.1) + .ok()?; + if !matches!(&self.tokens[i].0, Token::Word(w) if w.keyword == Keyword::SELECT) { + return None; + } + i += 1; + let significant = |t: &Token| !matches!(t, Token::Whitespace(_)); + while i < self.tokens.len() && !significant(&self.tokens[i].0) { + i += 1; + } + if matches!(&self.tokens.get(i)?.0, + Token::Word(w) if matches!(w.keyword, Keyword::DISTINCT | Keyword::ALL)) + { + i += 1; + } + + let mut items = Vec::new(); + let mut first: Option = None; + let mut last = 0; + let mut depth = 0usize; + let mut prev_distinct = false; + let mut push = |first: &mut Option, last: usize| -> Option<()> { + let f = (*first)?; + items.push(&self.text[self.tokens[f].2..self.tokens[last].3]); + *first = None; + Some(()) + }; + while let Some((tok, ..)) = self.tokens.get(i) { + if !significant(tok) { + i += 1; + continue; + } + match tok { + Token::LParen | Token::LBracket | Token::LBrace => depth += 1, + Token::RParen | Token::RBracket | Token::RBrace => { + if depth == 0 { + break; + } + depth -= 1; + } + Token::Comma if depth == 0 => { + push(&mut first, last)?; + i += 1; + continue; + } + Token::SemiColon if depth == 0 => break, + Token::Word(w) if depth == 0 => { + let ends = match w.keyword { + // `a IS [NOT] DISTINCT FROM b` stays in the item. + Keyword::FROM => !prev_distinct, + Keyword::WHERE + | Keyword::GROUP + | Keyword::HAVING + | Keyword::WINDOW + | Keyword::QUALIFY + | Keyword::ORDER + | Keyword::LIMIT + | Keyword::OFFSET + | Keyword::UNION + | Keyword::INTERSECT + | Keyword::EXCEPT + | Keyword::INTO => w.quote_style.is_none(), + _ => false, + }; + if ends { + break; + } + } + _ => {} + } + prev_distinct = matches!(tok, Token::Word(w) if w.keyword == Keyword::DISTINCT); + first.get_or_insert(i); + last = i; + i += 1; + } + push(&mut first, last)?; + (items.len() == select.projection.len()).then_some(items) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + /// The item texts of the first SELECT of `sql`, and of the SELECT of the + /// scalar subquery that is its first item when there is one. + fn texts(sql: &str) -> Vec { + let source = SqlSource::new(sql); + let Statement::Query(q) = one(sql) else { + panic!("not a query") + }; + let SetExpr::Select(sel) = q.body.as_ref() else { + panic!("not a select") + }; + let mut out: Vec = source + .select_item_texts(sel) + .expect("located") + .into_iter() + .map(String::from) + .collect(); + if let SelectItem::UnnamedExpr(Expr::Subquery(sub)) = &sel.projection[0] { + if let SetExpr::Select(inner) = sub.body.as_ref() { + out.extend( + source + .select_item_texts(inner) + .expect("inner") + .into_iter() + .map(String::from), + ); + } + } + out + } + + #[test] + fn select_item_texts_are_the_source_spans() { + assert_eq!( + texts("SELECT n + 1, COUNT( * ) FROM t"), + ["n + 1", "COUNT( * )"] + ); + assert_eq!( + texts("select distinct -a /* c */ , cast(a as text)\n from t where x"), + ["-a", "cast(a as text)"] + ); + assert_eq!( + texts("SELECT a IS NOT DISTINCT FROM 5, b AS x FROM t"), + ["a IS NOT DISTINCT FROM 5", "b AS x"] + ); + assert_eq!( + texts("SELECT 'é,', f(a, b) OVER (PARTITION BY c, d)"), + ["'é,'", "f(a, b) OVER (PARTITION BY c, d)"] + ); + assert_eq!( + texts("SELECT (SELECT max(v), 2\nFROM u) FROM t ORDER BY 1"), + ["(SELECT max(v), 2\nFROM u)", "max(v)", "2"] + ); + } + + fn one(sql: &str) -> Statement { + parse_sql(sql).unwrap().pop().unwrap() + } + + #[test] + fn compound_is_left_deep_in_source_order() { + let s = one("SELECT 1 UNION ALL SELECT 2 INTERSECT SELECT 3 EXCEPT SELECT 4"); + assert_eq!( + s.to_string(), + "SELECT 1 UNION ALL SELECT 2 INTERSECT SELECT 3 EXCEPT SELECT 4" + ); + let Statement::Query(q) = s else { panic!() }; + let SetExpr::SetOperation { left, op, .. } = *q.body else { + panic!() + }; + assert_eq!(op, SetOperator::Except); + let SetExpr::SetOperation { left, op, .. } = *left else { + panic!() + }; + assert_eq!(op, SetOperator::Intersect); + assert!(matches!( + *left, + SetExpr::SetOperation { + op: SetOperator::Union, + .. + } + )); + } + + #[test] + fn parenthesized_compound_operand_is_kept() { + let s = one("SELECT 1 UNION (SELECT 2 UNION SELECT 3) INTERSECT SELECT 4"); + let Statement::Query(q) = s else { panic!() }; + let SetExpr::SetOperation { + left, right, op, .. + } = *q.body + else { + panic!() + }; + assert_eq!(op, SetOperator::Intersect); + assert!(matches!(*right, SetExpr::Select(_))); + let SetExpr::SetOperation { right, .. } = *left else { + panic!() + }; + assert!(matches!(*right, SetExpr::Query(_))); + } + + #[test] + fn precedence_matches_sqlite() { + assert!(matches!( + expr_of("2 = 1 < 2"), + Expr::BinaryOp { + op: sqlparser::ast::BinaryOperator::Eq, + .. + } + )); + assert!(matches!( + expr_of("2 * 3 || 4"), + Expr::BinaryOp { + op: sqlparser::ast::BinaryOperator::Multiply, + .. + } + )); + assert!(matches!( + expr_of("0 IS NOT DISTINCT FROM 0 AND 0"), + Expr::BinaryOp { + op: sqlparser::ast::BinaryOperator::And, + .. + } + )); + assert!(matches!( + expr_of("'a' LIKE 'A' = 1"), + Expr::BinaryOp { + op: sqlparser::ast::BinaryOperator::Eq, + .. + } + )); + assert!(matches!( + expr_of("-1 || 2"), + Expr::BinaryOp { + op: sqlparser::ast::BinaryOperator::StringConcat, + .. + } + )); + assert!(matches!( + expr_of("1 < 2 = 2 > 1"), + Expr::BinaryOp { + op: sqlparser::ast::BinaryOperator::Eq, + .. + } + )); + } + + fn expr_of(sql: &str) -> Expr { + let Statement::Query(q) = one(&format!("SELECT {sql}")) else { + panic!() + }; + let SetExpr::Select(sel) = *q.body else { + panic!() + }; + match sel.projection.into_iter().next().unwrap() { + SelectItem::UnnamedExpr(e) => e, + _ => panic!(), + } + } + + #[test] + fn materialized_hints_are_dropped_only_in_a_with_list() { + let s = one("WITH a AS MATERIALIZED (SELECT 1), b AS NOT MATERIALIZED (SELECT 2) SELECT 1"); + let Statement::Query(q) = s else { panic!() }; + assert_eq!(q.with.expect("a WITH").cte_tables.len(), 2); + // A type name spelled MATERIALIZED is not a hint. + let s = one("SELECT CAST(1 AS materialized), (WITH c AS (SELECT 1) SELECT * FROM c)"); + assert_eq!( + s.to_string(), + "SELECT CAST(1 AS materialized), (WITH c AS (SELECT 1) SELECT * FROM c)" + ); + } + + #[test] + fn limits_reach_cte_bodies() { + let deep = format!( + "WITH c AS (SELECT {}) SELECT * FROM c", + vec!["1"; 1001].join("+") + ); + assert!(parse_sql(&deep).is_err()); + let compound = format!( + "WITH c AS ({}) SELECT * FROM c", + vec!["SELECT 1"; 501].join(" UNION ALL ") + ); + assert!(parse_sql(&compound).is_err()); + } + + #[test] + fn limits() { + let chain = |n: usize| format!("SELECT {}", vec!["1"; n].join("+")); + assert!(parse_sql(&chain(1000)).is_ok()); + let err = parse_sql(&chain(1001)).unwrap_err().to_string(); + assert!(err.contains("expression tree is too large (maximum depth 1000)")); + let compound = |n: usize| vec!["SELECT 1"; n].join(" UNION ALL "); + assert!(parse_sql(&compound(500)).is_ok()); + let err = parse_sql(&compound(501)).unwrap_err().to_string(); + assert!(err.contains("too many terms in compound SELECT")); + // Inside a subquery, nesting continues to count. + let sub = format!("SELECT (SELECT {})", vec!["1"; 1000].join("+")); + assert!(parse_sql(&sub).is_err()); + // As in SQLite, parentheses add no depth: `-(chain)` is the chain + // plus one. + let neg = |n: usize| format!("SELECT -({})", vec!["1"; n].join("+")); + assert!(parse_sql(&neg(999)).is_ok()); + assert!(parse_sql(&neg(1000)).is_err()); + } +} diff --git a/src/vm/prepass.rs b/src/vm/prepass.rs new file mode 100644 index 0000000..88f0f54 --- /dev/null +++ b/src/vm/prepass.rs @@ -0,0 +1,1312 @@ +//! The pre-pass run on every statement before it compiles: derived tables +//! and common table expressions become ordinary named tables. +//! +//! The compiler only knows how to open a cursor on a named table. So the +//! pre-pass walks the statement ([`parse::for_each_child`], the same walk +//! the parser's limit check uses, so it reaches every query, FROM item and +//! expression) and, for each FROM item that is +//! +//! - a derived table `(SELECT ...) d`: walks its query, compiles it, +//! registers the result under an internal name (`sqawk_derived_N`) and +//! rewrites the item to that table with alias `d`; +//! - a one-part name `c` that a CTE in scope declares: compiles the CTE +//! (once, on its first reference), registers it as `sqawk_cte_N` and +//! rewrites the item to that table, aliased as written or else as `c`, so +//! `c.col` works and the internal name never reaches the user; +//! - any other one-part name that some CTE of the statement declares: spells +//! it `main.name`, the loaded table. +//! +//! A registered table's rows are computed the first time the statement +//! opens it, by the compiled query (`Database::add_lazy_table`), as SQLite +//! computes a CTE or FROM subquery where it is first read: one read only by +//! code that never runs (a compound arm whose WHERE is false, a CASE branch +//! not taken) costs nothing. A recursive CTE is computed here. +//! +//! The compiler is then given the set of CTE names, and a one-part table +//! name from that set is an internal error there (see +//! `SqlCompiler::cte_names`): a reference this walk missed can never +//! silently read the loaded table of the same name. +//! +//! CTE scoping follows SQLite: a WITH list's CTEs see every sibling (those +//! declared after them too) and themselves; an inner WITH shadows an outer one; a CTE body +//! resolves names in the scope of its WITH, not of its reference; an unused +//! CTE is never evaluated. A CTE whose body references itself directly in a +//! UNION [ALL] arm is recursive (with or without RECURSIVE) and is computed +//! here, one queue row at a time, as SQLite does. +//! +//! Every table registered lives until the statement finishes (the caller +//! drops [`Prepared::registered`]). A derived table or CTE that refers to a +//! column of an enclosing query cannot be computed up front; it fails to +//! resolve ("no such column"). + +use std::cell::RefCell; +use std::collections::{BinaryHeap, HashSet, VecDeque}; +use std::rc::Rc; +use std::sync::Arc; + +use sqlparser::ast::{ + Cte, Expr, GroupByExpr, Ident, LimitClause, ObjectName, ObjectNamePart, OrderByKind, Query, + Select, SelectItem, SetExpr, SetOperator, SetQuantifier, Statement, TableAlias, TableFactor, + TableObject, Values, With, +}; + +use super::bytecode::{KeyOrder, Program, Register, ResultSchema}; +use super::compiler::SqlCompiler; +use super::compiler_check::check_order_by_expr; +use super::compiler_select::compound_order_key; +use super::compiler_window::contains_window; +use super::engine::{self, RowKey, VmEngine}; +use super::parse::{self, for_each_child, Node, SqlSource}; +use crate::database::Database; +use crate::error::{SqawkError, SqawkResult}; +use crate::table::{ColumnDefinition, Row, Table}; + +/// The outcome of the pre-pass over one statement. +pub(super) struct Prepared { + /// The tables registered for the statement, to drop when it finishes. + pub registered: Vec, + /// The CTE names (lower case) the statement's compiler must refuse as + /// one-part table names: those declared in the statement, plus those of + /// the enclosing statement when this one is a derived table or CTE body. + pub cte_names: Arc>, +} + +/// Materialize the derived tables and CTEs of `statement` and rewrite it +/// to reference them by name (see the module documentation). A `WITH ... +/// INSERT / UPDATE / DELETE` is unwrapped to its DML statement. +/// +/// `outer` holds the CTE names of the enclosing statement, when this one is +/// a derived table's or CTE's query. +pub(super) fn run( + statement: &mut Statement, + database: &mut Database, + verbose: bool, + source: &Arc, + outer: &Arc>, +) -> SqawkResult { + let mut declared = HashSet::new(); + collect_cte_names(Node::Statement(statement), &mut declared); + let cte_names = if declared.is_empty() { + Arc::clone(outer) + } else { + Arc::new(declared.iter().chain(outer.iter()).cloned().collect()) + }; + let mut walker = Walker { + database, + verbose, + source: Arc::clone(source), + declared, + cte_names: Arc::clone(&cte_names), + registered: Vec::new(), + scopes: Vec::new(), + }; + if let Err(e) = walker.visit(Node::Statement(statement)) { + for name in &walker.registered { + walker.database.remove_table(name); + } + return Err(e); + } + let registered = walker.registered; + unwrap_with_dml(statement); + Ok(Prepared { + registered, + cte_names, + }) +} + +/// `WITH ... INSERT / UPDATE / DELETE` parses as a query whose body is the +/// DML statement; once the WITH is resolved, the statement is the DML. +fn unwrap_with_dml(statement: &mut Statement) { + let Statement::Query(q) = statement else { + return; + }; + if q.with.is_some() || q.order_by.is_some() || q.limit_clause.is_some() { + return; + } + if !matches!( + *q.body, + SetExpr::Insert(_) | SetExpr::Update(_) | SetExpr::Delete(_) + ) { + return; + } + let body = std::mem::replace(&mut q.body, Box::new(empty_body())); + if let SetExpr::Insert(dml) | SetExpr::Update(dml) | SetExpr::Delete(dml) = *body { + *statement = dml; + } +} + +fn empty_body() -> SetExpr { + SetExpr::Values(Values { + explicit_row: false, + value_keyword: false, + rows: Vec::new(), + }) +} + +/// Every CTE name declared anywhere in the statement, lower case. +fn collect_cte_names(node: Node<'_>, out: &mut HashSet) { + if let Node::Query(q) = &node { + for cte in q.with.iter().flat_map(|w| &w.cte_tables) { + out.insert(key_of(&cte.alias.name)); + } + } + for_each_child(node, &mut |c| collect_cte_names(c, out)); +} + +/// How names are matched: case-insensitively, as SQL identifiers. +fn key_of(ident: &Ident) -> String { + ident.value.to_ascii_lowercase() +} + +/// The rows of a CTE or derived table, as registered: a table with the +/// result columns and, unless the rows are already in it, the program that +/// computes them on first use (see `Database::add_lazy_table`). +struct Rows { + table: Table, + program: Option, +} + +/// The error prefix for a reference to a CTE that is being computed. +const CIRCULAR: &str = "circular reference: "; +/// ... from a subquery of a recursive CTE's recursive arm. +const MULTIPLE_RECURSIVE: &str = "multiple recursive references: "; + +#[derive(Clone)] +enum CteState { + /// Not referenced yet. + Pending, + /// Being computed: a reference now is the error `prefix` + name. + InProgress(&'static str), + /// Materialized as this registered table. + Done(String), +} + +struct CteDef { + /// The name as declared. + name: Ident, + /// The column list, if any. + columns: Vec, + /// The body; taken when the CTE is computed. + query: Option>, + state: CteState, +} + +/// The CTEs of one WITH list. +struct Scope { + ctes: Vec, +} + +impl Scope { + fn new(with: With) -> SqawkResult { + let mut ctes: Vec = Vec::with_capacity(with.cte_tables.len()); + for cte in with.cte_tables { + let Cte { + alias, + query, + from, + materialized: _, + closing_paren_token: _, + } = cte; + if ctes + .iter() + .any(|c| c.name.value.eq_ignore_ascii_case(&alias.name.value)) + { + return Err(SqawkError::InvalidSqlQuery(format!( + "duplicate WITH table name: {}", + alias.name.value + ))); + } + if from.is_some() { + return Err(SqawkError::UnsupportedSqlFeature( + "WITH ... FROM is not supported".into(), + )); + } + if alias.columns.iter().any(|c| c.data_type.is_some()) { + return Err(SqawkError::UnsupportedSqlFeature( + "Typed CTE columns are not supported".into(), + )); + } + ctes.push(CteDef { + name: alias.name, + columns: alias.columns.into_iter().map(|c| c.name).collect(), + query: Some(query), + state: CteState::Pending, + }); + } + Ok(Scope { ctes }) + } +} + +struct Walker<'d> { + database: &'d mut Database, + verbose: bool, + source: Arc, + /// The CTE names declared in this statement: another one-part table + /// reference with one of these names is spelled `main.name`. + declared: HashSet, + /// What the compilers of this statement's queries refuse (see + /// [`Prepared::cte_names`]). + cte_names: Arc>, + registered: Vec, + /// The WITH lists in scope, innermost last. + scopes: Vec>>, +} + +impl Walker<'_> { + fn visit(&mut self, node: Node<'_>) -> SqawkResult<()> { + match node { + Node::Query(q) => self.visit_query(q), + Node::Factor(f) => self.visit_factor(f), + Node::Statement(s) => { + self.qualify_targets(s); + self.children(Node::Statement(s)) + } + Node::SetExpr(s) => { + reject_with_in_compound_operand(s)?; + self.children(Node::SetExpr(s)) + } + other => self.children(other), + } + } + + fn children(&mut self, node: Node<'_>) -> SqawkResult<()> { + let mut result = Ok(()); + for_each_child(node, &mut |c| { + if result.is_ok() { + result = self.visit(c); + } + }); + result + } + + /// A query's WITH list is in scope for its body, ORDER BY and LIMIT. + /// It is taken off the query: the compiler never sees it, and an unused + /// CTE's body is never walked. + fn visit_query(&mut self, q: &mut Query) -> SqawkResult<()> { + let Some(with) = q.with.take() else { + return self.children(Node::Query(q)); + }; + self.scopes.push(Rc::new(RefCell::new(Scope::new(with)?))); + let result = self.children(Node::Query(q)); + self.scopes.pop(); + result + } + + /// The table a DML statement or CREATE TABLE ... AS writes is always a + /// loaded table: spelled `main.name` when a CTE shares its name. + fn qualify_targets(&self, statement: &mut Statement) { + match statement { + Statement::Insert(insert) => { + if let TableObject::TableName(name) = &mut insert.table { + self.qualify(name); + } + } + Statement::Update(update) => { + if let TableFactor::Table { name, .. } = &mut update.table.relation { + self.qualify(name); + } + } + Statement::Delete(delete) => { + for name in &mut delete.tables { + self.qualify(name); + } + let (sqlparser::ast::FromTable::WithFromKeyword(from) + | sqlparser::ast::FromTable::WithoutKeyword(from)) = &mut delete.from; + for t in from { + if let TableFactor::Table { name, .. } = &mut t.relation { + self.qualify(name); + } + } + } + Statement::CreateTable(create) => self.qualify(&mut create.name), + _ => {} + } + } + + /// Spell a one-part `name` that some CTE of the statement declares as + /// `main.name`. + fn qualify(&self, name: &mut ObjectName) { + if let [part] = name.0.as_slice() { + if part + .as_ident() + .is_some_and(|i| self.declared.contains(&key_of(i))) + { + name.0 + .insert(0, ObjectNamePart::Identifier(Ident::new("main"))); + } + } + } + + fn visit_factor(&mut self, factor: &mut TableFactor) -> SqawkResult<()> { + match factor { + TableFactor::Table { + name, + alias, + args: None, + .. + } => { + let Some(ident) = one_part(name).cloned() else { + return Ok(()); + }; + let key = key_of(&ident); + match self.lookup(&key) { + Some((k, i)) => { + let table = self.cte_table(k, i)?; + let alias = alias.take().unwrap_or(TableAlias { + explicit: false, + name: ident, + columns: Vec::new(), + at: None, + }); + *factor = table_factor(table, Some(alias)); + } + None => self.qualify(name), + } + Ok(()) + } + TableFactor::Derived { lateral: true, .. } => Err(SqawkError::UnsupportedSqlFeature( + "LATERAL subqueries are not supported".into(), + )), + TableFactor::Derived { .. } => self.materialize_derived(factor), + _ => self.children(Node::Factor(factor)), + } + } + + /// The innermost CTE in scope named `key`: (scope index, CTE index). + fn lookup(&self, key: &str) -> Option<(usize, usize)> { + self.scopes.iter().enumerate().rev().find_map(|(k, s)| { + s.borrow() + .ctes + .iter() + .position(|c| key_of(&c.name) == key) + .map(|i| (k, i)) + }) + } + + /// Run a derived table's query and replace the FROM item with the + /// registered result, keeping its alias. + fn materialize_derived(&mut self, factor: &mut TableFactor) -> SqawkResult<()> { + let TableFactor::Derived { + subquery, alias, .. + } = factor + else { + return Ok(()); + }; + if alias.as_ref().is_some_and(|a| !a.columns.is_empty()) { + return Err(SqawkError::UnsupportedSqlFeature( + "Column aliases on a FROM item are not supported".into(), + )); + } + let alias = alias.take(); + let mut inner = std::mem::replace(subquery, Box::new(empty_query())); + // Nested derived tables and CTE references first. + self.visit_query(&mut inner)?; + let mut rows = self.compile_lazy(*inner)?; + rows.table.dedup_column_names(); + let name = self.register_rows(rows, "sqawk_derived_")?; + *factor = table_factor(name, alias); + Ok(()) + } + + /// The registered table of the CTE `(k, i)`, computing it on first use. + fn cte_table(&mut self, k: usize, i: usize) -> SqawkResult { + let (name, state) = { + let scope = self.scopes[k].borrow(); + let cte = &scope.ctes[i]; + (cte.name.clone(), cte.state.clone()) + }; + match state { + CteState::Done(table) => Ok(table), + CteState::InProgress(prefix) => Err(SqawkError::InvalidSqlQuery(format!( + "{}{}", + prefix, name.value + ))), + CteState::Pending => { + let (columns, query) = { + let mut scope = self.scopes[k].borrow_mut(); + let cte = &mut scope.ctes[i]; + cte.state = CteState::InProgress(CIRCULAR); + ( + cte.columns.clone(), + cte.query.take().expect("a pending CTE has its body"), + ) + }; + // The body sees the WITH lists up to and including its own, + // not those around the reference. + let inner = self.scopes[..=k].to_vec(); + let saved = std::mem::replace(&mut self.scopes, inner); + let result = self.evaluate_cte(&name, &columns, *query, k, i); + self.scopes = saved; + let mut rows = result?; + let table = &mut rows.table; + if !columns.is_empty() { + if columns.len() != table.column_count() { + return Err(SqawkError::InvalidSqlQuery(format!( + "table {} has {} values for {} columns", + name.value, + table.column_count(), + columns.len() + ))); + } + let names: Vec = columns.iter().map(|c| c.value.clone()).collect(); + table.rename_columns(&names); + } + table.dedup_column_names(); + let registered = self.register_rows(rows, "sqawk_cte_")?; + self.scopes[k].borrow_mut().ctes[i].state = CteState::Done(registered.clone()); + Ok(registered) + } + } + } + + /// A CTE's rows: its compiled body, run on first use, or, when it is + /// recursive, the rows iterating it gives now. + fn evaluate_cte( + &mut self, + name: &Ident, + columns: &[Ident], + mut query: Query, + k: usize, + i: usize, + ) -> SqawkResult { + if matches!( + *query.body, + SetExpr::Insert(_) | SetExpr::Update(_) | SetExpr::Delete(_) + ) { + return Err(SqawkError::UnsupportedSqlFeature( + "A data-modifying statement in WITH is not supported".into(), + )); + } + let recursive = recursive_arm_count(&query.body, name)?; + if recursive == 0 { + self.visit_query(&mut query)?; + return self.compile_lazy(query); + } + // The body's own WITH list is in scope for every arm. + let with = query.with.take(); + let pushed = match with { + Some(with) => { + self.scopes.push(Rc::new(RefCell::new(Scope::new(with)?))); + true + } + None => false, + }; + let result = self.evaluate_recursive(name, columns, query, recursive, k, i); + if pushed { + self.scopes.pop(); + } + Ok(Rows { + table: result?, + program: None, + }) + } + + /// SQLite's recursive CTE: the setup arms' rows seed a queue; each row + /// taken from it is output (subject to OFFSET and LIMIT) and then is the + /// whole content of the CTE for one run of every recursive arm, whose + /// rows join the queue. UNION keeps a row out of the queue once seen. + /// The queue is FIFO, or ordered by the body's ORDER BY (ties FIFO). + fn evaluate_recursive( + &mut self, + name: &Ident, + columns: &[Ident], + mut shell: Query, + recursive: usize, + k: usize, + i: usize, + ) -> SqawkResult
{ + let body = std::mem::replace(&mut shell.body, Box::new(empty_body())); + let order_by = shell.order_by.take(); + let limit_clause = shell.limit_clause.take(); + let (mut arms, ops) = flatten_compound(*body); + let distinct = matches!( + ops.last(), + Some((_, SetQuantifier::None | SetQuantifier::Distinct)) + ); + let recursive_arms = arms.split_off(arms.len() - recursive); + let mut setup = arms.into_iter(); + let mut setup_body = setup.next().expect("a compound has a first arm"); + for ((op, quantifier), arm) in ops.into_iter().zip(setup) { + setup_body = SetExpr::SetOperation { + left: Box::new(setup_body), + op, + set_quantifier: quantifier, + right: Box::new(arm), + }; + } + let with_body = |body: SetExpr| Query { + body: Box::new(body), + ..shell.clone() + }; + + // The setup arms: a reference to the CTE there is circular. + let mut setup_query = with_body(setup_body); + self.visit_query(&mut setup_query)?; + let mut table = self.run_query(setup_query)?; + let ncols = table.column_count(); + if !columns.is_empty() && columns.len() != ncols { + return Err(SqawkError::InvalidSqlQuery(format!( + "table {} has {} values for {} columns", + name.value, + ncols, + columns.len() + ))); + } + + // The body's ORDER BY names the setup's result columns. + let mut schema = ResultSchema::new(); + for col in table.column_metadata() { + schema.add_column(col.name.clone(), col.data_type); + } + let mut keys: Vec = Vec::new(); + match order_by.map(|ob| ob.kind) { + None => {} + Some(OrderByKind::All(_)) => { + return Err(SqawkError::UnsupportedSqlFeature( + "ORDER BY ALL is not supported".into(), + )) + } + Some(OrderByKind::Expressions(exprs)) => { + for (n, ob) in exprs.iter().enumerate() { + check_order_by_expr(ob)?; + keys.push(compound_order_key(ob, n, &schema)?); + } + } + } + let (limit, offset) = self.limits(limit_clause)?; + + // A FIFO queue is breadth-first: it holds the rows of one + // generation, each followed by its children, in order. So when every + // recursive arm maps each working row independently (no DISTINCT + // across rows) a whole generation can be the working table at once, + // each row tagged with its position: one run per generation instead + // of one per row, then the children are put back in queue order by + // (position, arm). A wildcard would expose the tag, so it keeps the + // row-at-a-time algorithm, as does an ordered queue. + let by_generation = keys.is_empty() + && recursive_arms.iter().all(|arm| { + matches!(arm, SetExpr::Select(s) + if s.distinct.is_none() + && s.projection.iter().all(|item| matches!(item, + SelectItem::UnnamedExpr(_) | SelectItem::ExprWithAlias { .. }))) + }); + + // The working table, under the CTE's column names (its column list, + // else the setup's), plus the position tag in generation mode. + let mut defs: Vec = table + .column_metadata() + .iter() + .enumerate() + .map(|(n, c)| ColumnDefinition { + name: columns + .get(n) + .map_or_else(|| c.name.clone(), |i| i.value.clone()), + data_type: c.data_type, + decl_type: None, + }) + .collect(); + if by_generation { + defs.push(ColumnDefinition { + name: POSITION.to_string(), + data_type: crate::table::DataType::Integer, + decl_type: None, + }); + } + let mut work = Table::new_with_schema("work", defs, None, None); + work.dedup_column_names(); + let work_name = self.register(work, "sqawk_cte_work_")?; + + // Any reference besides the one direct FROM item of each recursive + // arm is an error from here on. + self.scopes[k].borrow_mut().ctes[i].state = CteState::InProgress(MULTIPLE_RECURSIVE); + let op = if distinct { "UNION" } else { "UNION ALL" }; + let width = ncols + usize::from(by_generation); + let mut programs: Vec = Vec::with_capacity(recursive_arms.len()); + for mut arm in recursive_arms { + if let SetExpr::Select(select) = &mut arm { + let bound = bind_working_table(select, name, &work_name); + if by_generation { + select + .projection + .push(SelectItem::UnnamedExpr(Expr::CompoundIdentifier(vec![ + bound, + Ident::new(POSITION), + ]))); + } + } + let mut query = with_body(arm); + self.visit_query(&mut query)?; + programs.push(self.compile_recursive_arm(query, width, op)?); + } + + let setup_rows: Vec = table.rows().to_vec(); + let mut run = Recursion { + seen: HashSet::new(), + distinct, + limit, + offset, + out: Vec::new(), + }; + if limit != Some(0) { + if by_generation { + self.recurse_by_generation(&mut run, setup_rows, &work_name, &programs)?; + } else { + self.recurse_by_row(&mut run, setup_rows, keys, &work_name, &programs)?; + } + } + table.clear_rows()?; + for row in run.out { + table.add_row(row)?; + } + Ok(table) + } + + /// Run `program` over the database as it stands; its result rows. + fn run_program(&self, program: &Program) -> SqawkResult> { + let mut vm = VmEngine::new_mut(self.database, false); + vm.init(program.clone()); + vm.execute()?; + Ok(vm.take_results()) + } + + /// Make `rows` the content of the working table. + fn fill_working_table(&mut self, work: &str, rows: Vec) -> SqawkResult<()> { + let table = self.database.get_table_mut(work)?; + table.clear_rows()?; + for row in rows { + table.add_row(row)?; + } + Ok(()) + } + + /// SQLite's algorithm as written: one queue row at a time. + fn recurse_by_row( + &mut self, + run: &mut Recursion, + setup: Vec, + keys: Vec, + work: &str, + programs: &[Program], + ) -> SqawkResult<()> { + let mut queue = Queue::new(keys); + for row in setup { + if run.admit(&row) { + queue.push(row); + } + } + while let Some(row) = queue.pop() { + if run.emit(&row) { + break; + } + self.fill_working_table(work, vec![row])?; + for program in programs { + for r in self.run_program(program)? { + if run.admit(&r) { + queue.push(r); + } + } + } + } + Ok(()) + } + + /// The FIFO algorithm one generation at a time (see + /// `evaluate_recursive`): the same rows in the same order. + fn recurse_by_generation( + &mut self, + run: &mut Recursion, + setup: Vec, + work: &str, + programs: &[Program], + ) -> SqawkResult<()> { + let mut generation: Vec = setup.into_iter().filter(|r| run.admit(r)).collect(); + while !generation.is_empty() { + for row in &generation { + if run.emit(row) { + return Ok(()); + } + } + let tagged = generation + .into_iter() + .enumerate() + .map(|(n, mut row)| { + row.push(crate::table::Value::Integer(n as i64)); + row + }) + .collect(); + self.fill_working_table(work, tagged)?; + let mut children: Vec<(i64, usize, Row)> = Vec::new(); + for (arm, program) in programs.iter().enumerate() { + for mut r in self.run_program(program)? { + let Some(crate::table::Value::Integer(n)) = r.pop() else { + return Err(SqawkError::VmError( + "internal error: recursive row lost its position".into(), + )); + }; + children.push((n, arm, r)); + } + } + // Stable: within one parent and arm, the rows keep their order. + children.sort_by_key(|(n, arm, _)| (*n, *arm)); + generation = children + .into_iter() + .map(|(.., r)| r) + .filter(|r| run.admit(r)) + .collect(); + } + Ok(()) + } + + /// Compile one recursive arm, after SQLite's checks on it. + fn compile_recursive_arm( + &mut self, + query: Query, + ncols: usize, + op: &str, + ) -> SqawkResult { + let mut compiler = SqlCompiler::new(self.database, self.verbose) + .with_source(Some(Arc::clone(&self.source)), true) + .with_cte_names(Arc::clone(&self.cte_names)); + if let SetExpr::Select(select) = &*query.body { + let exprs = || select_items(select); + if exprs().any(contains_window) { + return Err(SqawkError::InvalidSqlQuery( + "cannot use window functions in recursive queries".into(), + )); + } + let grouped = + !matches!(&select.group_by, GroupByExpr::Expressions(e, _) if e.is_empty()); + if grouped || select.having.is_some() || exprs().any(|e| compiler.is_aggregate_expr(e)) + { + return Err(SqawkError::InvalidSqlQuery( + "recursive aggregate queries not supported".into(), + )); + } + } + let program = compiler.compile_statement_program(&Statement::Query(Box::new(query)))?; + if program.result_schema.columns.len() != ncols { + return Err(SqawkError::InvalidSqlQuery(format!( + "SELECTs to the left and right of {} do not have the same number of result columns", + op + ))); + } + Ok(program) + } + + /// A recursive CTE body's LIMIT and OFFSET, evaluated once: `None` for + /// no limit (absent or negative), and an offset of at least 0. + fn limits(&mut self, clause: Option) -> SqawkResult<(Option, usize)> { + let (limit, offset) = match clause { + None => (None, None), + Some(LimitClause::LimitOffset { + limit, + offset, + limit_by, + }) => { + if !limit_by.is_empty() { + return Err(SqawkError::UnsupportedSqlFeature( + "LIMIT ... BY is not supported".into(), + )); + } + (limit, offset.map(|o| o.value)) + } + Some(LimitClause::OffsetCommaLimit { offset, limit }) => (Some(limit), Some(offset)), + }; + let limit = match limit { + Some(e) => { + let n = self.eval_integer(e, "LIMIT")?; + usize::try_from(n).ok() + } + None => None, + }; + let offset = match offset { + Some(e) => usize::try_from(self.eval_integer(e, "OFFSET")?).unwrap_or(0), + None => 0, + }; + Ok((limit, offset)) + } + + /// Evaluate a constant expression (it may contain subqueries) as a + /// LIMIT / OFFSET operand. + fn eval_integer(&mut self, expr: Expr, what: &str) -> SqawkResult { + let mut statement = parse::parse_sql("SELECT NULL")? + .pop() + .expect("one statement"); + if let Statement::Query(q) = &mut statement { + if let SetExpr::Select(select) = q.body.as_mut() { + select.projection = vec![SelectItem::UnnamedExpr(expr)]; + } + } + self.visit(Node::Statement(&mut statement))?; + let Statement::Query(q) = statement else { + unreachable!("a SELECT") + }; + let table = self.run_query(*q)?; + let value = table + .rows() + .first() + .and_then(|r| r.first()) + .cloned() + .unwrap_or(crate::table::Value::Null); + engine::must_be_int(Register::from(value), what) + } + + /// Run a query (already walked) and return its result. + fn run_query(&mut self, query: Query) -> SqawkResult
{ + let source = Arc::clone(&self.source); + let result = super::execute_statements( + vec![Statement::Query(Box::new(query))], + self.database, + self.verbose, + &source, + true, + &self.cte_names, + )?; + result.tables.into_iter().next_back().ok_or_else(|| { + SqawkError::InvalidSqlQuery("Subquery in FROM produced no result".to_string()) + }) + } + + /// Compile a query (already walked) whose rows are computed when the + /// statement first reads them: its result columns as a table with no + /// rows, and the program. + fn compile_lazy(&mut self, query: Query) -> SqawkResult { + let mut compiler = SqlCompiler::new(self.database, self.verbose) + .with_source(Some(Arc::clone(&self.source)), true) + .with_cte_names(Arc::clone(&self.cte_names)); + let program = compiler.compile_statement_program(&Statement::Query(Box::new(query)))?; + if program.result_schema.is_empty() { + return Err(SqawkError::InvalidSqlQuery( + "Subquery in FROM produced no result".to_string(), + )); + } + let defs = program + .result_schema + .columns + .iter() + .map(|col| ColumnDefinition { + name: col.name.clone(), + data_type: col.data_type, + decl_type: None, + }) + .collect(); + Ok(Rows { + table: Table::new_with_schema("result", defs, None, None), + program: Some(program), + }) + } + + /// Register `rows` under a fresh internal name starting `prefix`. + fn register_rows(&mut self, rows: Rows, prefix: &str) -> SqawkResult { + let Rows { mut table, program } = rows; + let Some(program) = program else { + return self.register(table, prefix); + }; + let name = self.fresh_name(prefix); + table.set_name(name.clone()); + if self.verbose { + diag!("Bytecode for {} (run on first use):\n{}", name, program); + } + self.database.add_lazy_table(name.clone(), table, program)?; + self.registered.push(name.clone()); + Ok(name) + } + + /// Register `table` under a fresh internal name starting `prefix`. + fn register(&mut self, mut table: Table, prefix: &str) -> SqawkResult { + let name = self.fresh_name(prefix); + table.set_name(name.clone()); + self.database.add_table(name.clone(), table)?; + self.registered.push(name.clone()); + Ok(name) + } + + /// The first internal table name `prefix`N not in use. + fn fresh_name(&self, prefix: &str) -> String { + (1..) + .map(|n| format!("{}{}", prefix, n)) + .find(|n| !self.database.has_table(n)) + .expect("an unused name") + } +} + +/// WITH on a parenthesized compound operand is a syntax error in SQLite; +/// sqawk keeps rejecting it rather than resolve it. +fn reject_with_in_compound_operand(s: &SetExpr) -> SqawkResult<()> { + if let SetExpr::SetOperation { left, right, .. } = s { + for operand in [left, right] { + if matches!(&**operand, SetExpr::Query(q) if q.with.is_some()) { + return Err(SqawkError::UnsupportedSqlFeature( + "ORDER BY, LIMIT or WITH inside a compound SELECT operand is not supported" + .into(), + )); + } + } + } + Ok(()) +} + +/// The name of a one-part table reference. +fn one_part(name: &ObjectName) -> Option<&Ident> { + match name.0.as_slice() { + [part] => part.as_ident(), + _ => None, + } +} + +/// A FROM item reading the registered table `name`. +fn table_factor(name: String, alias: Option) -> TableFactor { + TableFactor::Table { + name: ObjectName(vec![ObjectNamePart::Identifier(Ident::new(name))]), + alias, + args: None, + with_hints: Vec::new(), + version: None, + partitions: Vec::new(), + with_ordinality: false, + json_path: None, + sample: None, + index_hints: Vec::new(), + } +} + +fn empty_query() -> Query { + Query { + with: None, + body: Box::new(empty_body()), + order_by: None, + limit_clause: None, + fetch: None, + locks: Vec::new(), + for_clause: None, + settings: None, + format_clause: None, + pipe_operators: Vec::new(), + } +} + +/// UNION and UNION ALL are told apart; other quantifiers are their own op. +fn same_op(a: (&SetOperator, &SetQuantifier), b: (&SetOperator, &SetQuantifier)) -> bool { + let norm = |q: &SetQuantifier| match q { + SetQuantifier::None | SetQuantifier::Distinct => SetQuantifier::Distinct, + other => *other, + }; + a.0 == b.0 && norm(a.1) == norm(b.1) +} + +/// The number of recursive arms of a CTE body, as SQLite counts them: the +/// body must be a UNION [ALL] compound; walking its arms from the right +/// while they are joined by that same operator, each arm whose FROM names +/// the CTE directly is recursive, up to the first that does not. A +/// recursive arm may name it once. 0: not recursive. +fn recursive_arm_count(body: &SetExpr, name: &Ident) -> SqawkResult { + let SetExpr::SetOperation { + op: top_op, + set_quantifier: top_q, + .. + } = body + else { + return Ok(0); + }; + if *top_op != SetOperator::Union + || !matches!( + top_q, + SetQuantifier::None | SetQuantifier::Distinct | SetQuantifier::All + ) + { + return Ok(0); + } + let key = key_of(name); + let mut count = 0; + let mut node = body; + while let SetExpr::SetOperation { + op, + set_quantifier, + left, + right, + } = node + { + if !same_op((op, set_quantifier), (top_op, top_q)) { + break; + } + let refs = match &**right { + SetExpr::Select(select) => direct_references(select, &key), + _ => 0, + }; + if refs > 1 { + return Err(SqawkError::InvalidSqlQuery(format!( + "multiple references to recursive table: {}", + name.value + ))); + } + if refs == 0 { + break; + } + count += 1; + node = left; + } + Ok(count) +} + +/// FROM items of `select` (joins included, not nested) naming `key`. +fn direct_references(select: &Select, key: &str) -> usize { + select + .from + .iter() + .flat_map(|t| std::iter::once(&t.relation).chain(t.joins.iter().map(|j| &j.relation))) + .filter(|f| { + matches!(f, TableFactor::Table { name, args: None, .. } + if one_part(name).is_some_and(|i| key_of(i) == key)) + }) + .count() +} + +/// The working table's position column in generation mode. +const POSITION: &str = "sqawk_position"; + +/// The output side of a recursive CTE: the rows UNION has seen, LIMIT and +/// OFFSET, and the rows output so far. +struct Recursion { + seen: HashSet>, + distinct: bool, + limit: Option, + offset: usize, + out: Vec, +} + +impl Recursion { + /// Whether `row` joins the queue: under UNION, only the first time. + fn admit(&mut self, row: &Row) -> bool { + !self.distinct || self.seen.insert(row.iter().map(engine::row_key).collect()) + } + + /// Output a row taken from the queue, unless OFFSET skips it. True when + /// LIMIT is reached, which ends the recursion. + fn emit(&mut self, row: &Row) -> bool { + if self.offset > 0 { + self.offset -= 1; + return false; + } + self.out.push(row.clone()); + self.limit == Some(self.out.len()) + } +} + +/// Point a recursive arm's direct reference to the CTE at the working +/// table, keeping the name it is qualified by, which is returned. +fn bind_working_table(select: &mut Select, name: &Ident, work: &str) -> Ident { + let key = key_of(name); + let mut bound = name.clone(); + for t in &mut select.from { + for factor in + std::iter::once(&mut t.relation).chain(t.joins.iter_mut().map(|j| &mut j.relation)) + { + if let TableFactor::Table { + name: table, + alias, + args: None, + .. + } = factor + { + let Some(ident) = one_part(table).filter(|i| key_of(i) == key).cloned() else { + continue; + }; + let alias = alias.take().unwrap_or(TableAlias { + explicit: false, + name: ident, + columns: Vec::new(), + at: None, + }); + bound = alias.name.clone(); + *factor = table_factor(work.to_string(), Some(alias)); + } + } + } + bound +} + +/// The arms of a left-deep compound in order, and the operators between +/// them (`ops[j]` joins the first `j + 1` arms to arm `j + 1`). +fn flatten_compound(body: SetExpr) -> (Vec, Vec<(SetOperator, SetQuantifier)>) { + let mut arms = Vec::new(); + let mut ops = Vec::new(); + let mut node = body; + loop { + match node { + SetExpr::SetOperation { + left, + op, + set_quantifier, + right, + } => { + arms.push(*right); + ops.push((op, set_quantifier)); + node = *left; + } + other => { + arms.push(other); + break; + } + } + } + arms.reverse(); + ops.reverse(); + (arms, ops) +} + +/// The expressions of a SELECT that decide whether it aggregates or uses +/// windows: its result columns. +fn select_items(select: &Select) -> impl Iterator { + select.projection.iter().filter_map(|item| match item { + SelectItem::UnnamedExpr(e) | SelectItem::ExprWithAlias { expr: e, .. } => Some(e), + _ => None, + }) +} + +/// The queue of a recursive CTE: FIFO, or ordered by sort keys with ties in +/// arrival order. +enum Queue { + Fifo(VecDeque), + Ordered { + heap: BinaryHeap, + keys: Rc<[KeyOrder]>, + seq: u64, + }, +} + +impl Queue { + fn new(keys: Vec) -> Self { + if keys.is_empty() { + Queue::Fifo(VecDeque::new()) + } else { + Queue::Ordered { + heap: BinaryHeap::new(), + keys: keys.into(), + seq: 0, + } + } + } + + fn push(&mut self, row: Row) { + match self { + Queue::Fifo(q) => q.push_back(row), + Queue::Ordered { heap, keys, seq } => { + heap.push(Queued { + row, + seq: *seq, + keys: Rc::clone(keys), + }); + *seq += 1; + } + } + } + + fn pop(&mut self) -> Option { + match self { + Queue::Fifo(q) => q.pop_front(), + Queue::Ordered { heap, .. } => heap.pop().map(|q| q.row), + } + } +} + +struct Queued { + row: Row, + seq: u64, + keys: Rc<[KeyOrder]>, +} + +impl Ord for Queued { + /// Reversed, so the max-heap yields the smallest key, earliest first. + fn cmp(&self, other: &Self) -> std::cmp::Ordering { + engine::compare_rows(&self.keys, &other.row, &self.row).then(other.seq.cmp(&self.seq)) + } +} + +impl PartialOrd for Queued { + fn partial_cmp(&self, other: &Self) -> Option { + Some(self.cmp(other)) + } +} + +impl PartialEq for Queued { + fn eq(&self, other: &Self) -> bool { + self.cmp(other) == std::cmp::Ordering::Equal + } +} + +impl Eq for Queued {} + +#[cfg(test)] +mod tests { + use super::*; + use crate::table::Value; + + fn database() -> Database { + let mut db = Database::new(); + let mut emp = Table::new("emp", vec!["id".to_string()], None); + emp.add_row(vec![Value::Integer(1)]).unwrap(); + db.add_table("emp".to_string(), emp).unwrap(); + db + } + + fn one(sql: &str) -> Statement { + parse::parse_sql(sql).unwrap().pop().unwrap() + } + + fn names(list: &[&str]) -> Arc> { + Arc::new(list.iter().map(|s| s.to_string()).collect()) + } + + /// The compile-time guard: a one-part table name that is a CTE name of + /// the statement never reads the loaded table of that name. + #[test] + fn compiler_refuses_an_unresolved_cte_name() { + let db = database(); + let compile = |sql: &str| { + SqlCompiler::new(&db, false) + .with_cte_names(names(&["emp"])) + .compile_statement_program(&one(sql)) + }; + let err = compile("SELECT id FROM emp").unwrap_err().to_string(); + assert_eq!(err, "internal error: unresolved CTE emp"); + let err = compile("SELECT (SELECT 1 FROM EMP)") + .unwrap_err() + .to_string(); + assert_eq!(err, "internal error: unresolved CTE EMP"); + assert!(compile("SELECT id FROM main.emp").is_ok()); + } + + /// The pre-pass removes every WITH, points each CTE reference at the + /// CTE's registered table under the CTE's name, and spells the loaded + /// table `main.emp` wherever a CTE named `emp` is not in scope. + #[test] + fn rewrites_references() { + let mut db = database(); + let source = Arc::new(SqlSource::new("")); + let mut statement = one("SELECT (SELECT COUNT(*) FROM emp), \ + (WITH emp AS (SELECT 2 AS id) SELECT id FROM emp) FROM emp"); + let prepared = run(&mut statement, &mut db, false, &source, &Arc::default()).unwrap(); + assert_eq!(*prepared.cte_names, *names(&["emp"])); + assert_eq!(prepared.registered, ["sqawk_cte_1"]); + assert_eq!( + statement.to_string(), + "SELECT (SELECT COUNT(*) FROM main.emp), \ + (SELECT id FROM sqawk_cte_1 emp) FROM main.emp" + ); + } + + /// `WITH ... DELETE` becomes the DELETE. + #[test] + fn unwraps_with_dml() { + let mut db = database(); + let source = Arc::new(SqlSource::new("")); + let mut statement = + one("WITH r AS (SELECT 1 AS id) DELETE FROM emp WHERE id IN (SELECT id FROM r)"); + run(&mut statement, &mut db, false, &source, &Arc::default()).unwrap(); + assert!(matches!(statement, Statement::Delete(_))); + assert_eq!( + statement.to_string(), + "DELETE FROM emp WHERE id IN (SELECT id FROM sqawk_cte_1 r)" + ); + } +} diff --git a/src/vm/printf.rs b/src/vm/printf.rs new file mode 100644 index 0000000..12034b8 --- /dev/null +++ b/src/vm/printf.rs @@ -0,0 +1,809 @@ +//! SQLite's printf: the formatter behind PRINTF / FORMAT, QUOTE and the +//! numeric fields of the date functions. A port of `sqlite3_str_vappendf` +//! (printf.c, SQLite 3.45) as SQLite's SQL function drives it, conversion +//! for conversion, quirks included: +//! +//! - conversions `d i u f e E g G x X o c s z q Q w p r n %`; flags +//! `- + space # ! 0 ,`; width and precision, either one `*` (taken from +//! the arguments; a negative width left-justifies); the C length +//! modifiers `l` / `ll` are accepted and ignored; +//! - an unknown conversion, or a format ending inside a conversion, ends +//! the output there (what was formatted so far is the result, NULL when +//! that is nothing at all; an empty format is NULL too); +//! - a width or precision past the length limit is the error "string or +//! blob too big" (SQLite wraps it to 31 bits or returns NULL); +//! - reals are decoded by [`fp_decode`] (SQLite's double-double +//! `sqlite3FpDecode`, identical on every platform) to at most 16 +//! significant digits (26 with `!`), rounded half up -- so `%.20f` of 0.1 is +//! `0.10000000000000000000`, and `%.0f` truncates (`%.0f` of 2.5 is `2`); +//! - `%s` widths and precisions count bytes unless the `!` flag is given; +//! - a missing argument is NULL; NULL is 0 for numeric conversions, `''` +//! for `%s`, `(NULL)` for `%q` / `%w` and `NULL` for `%Q`. + +use super::arith::{fp_decode, real_to_int, text_of, text_to_int, text_to_num}; +use super::bytecode::Register; +use super::func::{too_big, MAX_LENGTH}; +use crate::error::SqawkResult; + +/// SQLite's SQLITE_FP_PRECISION_LIMIT: the largest precision a real +/// conversion honours. +const FP_PRECISION_LIMIT: i64 = 100_000_000; + +/// The arguments of one call, consumed in order (SQLite's +/// `PrintfArguments`). +struct Args<'a> { + args: &'a [Register], + used: usize, +} + +impl Args<'_> { + fn next(&mut self) -> Option<&Register> { + let r = self.args.get(self.used); + if r.is_some() { + self.used += 1; + } + r + } + + /// `sqlite3_value_int64`: text by its integer prefix, reals truncated. + fn int(&mut self) -> i64 { + match self.next() { + None | Some(Register::Null) => 0, + Some(Register::Integer(i)) => *i, + Some(Register::Boolean(b)) => *b as i64, + Some(Register::Float(f)) => real_to_int(*f), + Some(Register::String(s)) => text_to_int(s), + } + } + + /// `sqlite3_value_double`: text by its numeric prefix. + fn double(&mut self) -> f64 { + match self.next() { + None | Some(Register::Null) => 0.0, + Some(Register::Integer(i)) => *i as f64, + Some(Register::Boolean(b)) => *b as i64 as f64, + Some(Register::Float(f)) => *f, + Some(Register::String(s)) => text_to_num(s).as_f64(), + } + } + + /// `sqlite3_value_text`: `None` for NULL or a missing argument. + fn text(&mut self) -> Option> { + match self.next() { + None | Some(Register::Null) => None, + Some(r) => Some(text_of(r).into_bytes()), + } + } +} + +/// The output, never longer than [`MAX_LENGTH`]. `touched` records whether +/// anything was appended, even nothing: SQLite's result is NULL when no +/// append ever happened (an empty format, or one that stops at its first +/// conversion), but `''` after an empty one (`printf('%s', '')`). +struct Out { + buf: Vec, + touched: bool, +} + +impl Out { + fn push(&mut self, b: &[u8]) -> SqawkResult<()> { + if self.buf.len().saturating_add(b.len()) > MAX_LENGTH { + return Err(too_big()); + } + self.buf.extend_from_slice(b); + self.touched = true; + Ok(()) + } + + fn pad(&mut self, n: i64, ch: u8) -> SqawkResult<()> { + if n <= 0 { + return Ok(()); + } + let n = n as usize; + if self.buf.len().saturating_add(n) > MAX_LENGTH { + return Err(too_big()); + } + self.buf.resize(self.buf.len() + n, ch); + self.touched = true; + Ok(()) + } + + fn finish(self) -> Option> { + self.touched.then_some(self.buf) + } +} + +/// A width or precision written in the format, or taken from an argument, +/// past the length limit is an error (SQLite wraps it to 31 bits, or gives +/// NULL once the result is too long). +fn check_size(n: u64) -> SqawkResult<()> { + if n > MAX_LENGTH as u64 { + Err(too_big()) + } else { + Ok(()) + } +} + +/// Ok if a work buffer of `n` bytes is within the length limit (SQLite's +/// `printfTempBuf` fails with SQLITE_TOOBIG past it). +fn temp_buf(n: i64) -> SqawkResult<()> { + if n > MAX_LENGTH as i64 { + Err(too_big()) + } else { + Ok(()) + } +} + +#[derive(Clone, Copy, PartialEq, Eq)] +enum Kind { + Decimal, + Radix, + Ordinal, + Pointer, + Float, + Exp, + Generic, + Size, + Str, + SqlEscape, + SqlEscape2, + SqlEscape3, + CharX, + Percent, + /// `%T`, `%S` (internal to SQLite) and anything unknown: stop. + Stop, +} + +/// One conversion's table entry: kind, base, signed, upper-case digits, +/// and the `#` prefix. +struct Info { + kind: Kind, + base: u64, + signed: bool, + upper: bool, + alt_prefix: &'static [u8], +} + +fn info(c: u8) -> Info { + let i = |kind, base, signed, upper, alt_prefix| Info { + kind, + base, + signed, + upper, + alt_prefix, + }; + match c { + b'd' | b'i' => i(Kind::Decimal, 10, true, false, b""), + b'u' => i(Kind::Decimal, 10, false, false, b""), + b's' | b'z' => i(Kind::Str, 0, false, false, b""), + b'g' => i(Kind::Generic, 0, true, false, b""), + b'G' => i(Kind::Generic, 0, true, true, b""), + b'q' => i(Kind::SqlEscape, 0, false, false, b""), + b'Q' => i(Kind::SqlEscape2, 0, false, false, b""), + b'w' => i(Kind::SqlEscape3, 0, false, false, b""), + b'c' => i(Kind::CharX, 0, false, false, b""), + b'o' => i(Kind::Radix, 8, false, false, b"0"), + b'x' => i(Kind::Radix, 16, false, false, b"0x"), + b'X' => i(Kind::Radix, 16, false, true, b"0X"), + b'f' => i(Kind::Float, 0, true, false, b""), + b'e' => i(Kind::Exp, 0, true, false, b""), + b'E' => i(Kind::Exp, 0, true, true, b""), + b'n' => i(Kind::Size, 0, false, false, b""), + b'%' => i(Kind::Percent, 0, false, false, b""), + b'p' => i(Kind::Pointer, 16, false, true, b"0x"), + b'r' => i(Kind::Ordinal, 10, true, false, b""), + _ => i(Kind::Stop, 0, false, false, b""), + } +} + +/// `printf(fmt, args...)` as SQLite's SQL function formats it, as bytes +/// (a `%s` precision counts bytes, so it may cut a character in two); +/// `None` is SQL NULL. +pub(crate) fn format(fmt: &[u8], args: &[Register]) -> SqawkResult>> { + let mut args = Args { args, used: 0 }; + let mut out = Out { + buf: Vec::new(), + touched: false, + }; + let at = |k: usize| fmt.get(k).copied().unwrap_or(0); + let mut i = 0usize; + while i < fmt.len() { + if fmt[i] != b'%' { + let start = i; + while i < fmt.len() && fmt[i] != b'%' { + i += 1; + } + out.push(&fmt[start..i])?; + if i >= fmt.len() { + break; + } + } + // fmt[i] is '%'. + i += 1; + let mut c = at(i); + if c == 0 { + out.push(b"%")?; + break; + } + let mut left = false; + let mut prefix: u8 = 0; + let mut thousand = false; + let mut alt = false; + let mut alt2 = false; + let mut zeropad = false; + let mut width: i64 = 0; + let mut precision: i64 = -1; + loop { + let mut done = false; + match c { + b'-' => left = true, + b'+' => prefix = b'+', + b' ' => prefix = b' ', + b'#' => alt = true, + b'!' => alt2 = true, + b'0' => zeropad = true, + b',' => thousand = true, + b'l' => { + i += 1; + c = at(i); + if c == b'l' { + i += 1; + c = at(i); + } + done = true; + } + b'1'..=b'9' => { + let mut wx = u64::from(c - b'0'); + loop { + i += 1; + c = at(i); + if !c.is_ascii_digit() { + break; + } + wx = wx.saturating_mul(10).saturating_add(u64::from(c - b'0')); + } + check_size(wx)?; + width = wx as i64; + if c != b'.' && c != b'l' { + done = true; + } else { + i -= 1; + } + } + b'*' => { + let w = args.int(); + check_size(w.unsigned_abs())?; + width = w; + if w < 0 { + left = true; + width = -w; + } + let c1 = at(i + 1); + if c1 != b'.' && c1 != b'l' { + i += 1; + c = at(i); + done = true; + } + } + b'.' => { + i += 1; + c = at(i); + if c == b'*' { + // A negative precision counts as its magnitude. + let p = args.int(); + check_size(p.unsigned_abs())?; + precision = p.abs(); + i += 1; + c = at(i); + } else { + let mut px: u64 = 0; + while c.is_ascii_digit() { + px = px.saturating_mul(10).saturating_add(u64::from(c - b'0')); + i += 1; + c = at(i); + } + check_size(px)?; + precision = px as i64; + } + if c == b'l' { + i -= 1; + } else { + done = true; + } + } + _ => done = true, + } + if done { + break; + } + i += 1; + c = at(i); + if c == 0 { + break; + } + } + // fmt[i] is the conversion character (or the end of the format). + i += 1; + let inf = info(c); + let mut body: Vec = Vec::new(); + match inf.kind { + Kind::Stop => return Ok(out.finish()), + Kind::Size => width = 0, + Kind::Percent => body.push(b'%'), + Kind::Decimal | Kind::Radix | Kind::Ordinal | Kind::Pointer => { + if inf.kind != Kind::Decimal { + thousand = false; + } + let (longvalue, sign) = if inf.signed { + let v = args.int(); + if v < 0 { + (v.unsigned_abs(), b'-') + } else { + (v as u64, prefix) + } + } else { + (args.int() as u64, 0) + }; + body = integer( + longvalue, sign, &inf, precision, width, zeropad, alt, thousand, + )?; + } + Kind::Float | Kind::Exp | Kind::Generic => { + let v = args.double(); + body = real( + v, inf.kind, inf.upper, precision, width, prefix, left, zeropad, alt, alt2, + thousand, + )?; + } + Kind::CharX => { + let mut ch: Vec = vec![0]; + if let Some(t) = args.text() { + if let Some(&first) = t.first() { + ch[0] = first; + if first & 0xc0 == 0xc0 { + ch.extend(t[1..].iter().take(3).take_while(|&&b| b & 0xc0 == 0x80)); + } + } + } + if precision > 1 { + width -= precision - 1; + if width > 1 && !left { + out.pad(width - 1, b' ')?; + width = 0; + } + let reps = (precision - 1) as usize; + if (ch.len() as u64).saturating_mul(reps as u64) > MAX_LENGTH as u64 { + return Err(too_big()); + } + out.push(&ch.repeat(reps))?; + } + width += utf8_extra(&ch, true, width); + body = ch; + } + Kind::Str => { + let t = args.text().unwrap_or_default(); + // The text ends at a NUL, as a C string does. + let t = match t.iter().position(|&b| b == 0) { + Some(n) => t[..n].to_vec(), + None => t, + }; + let len = if precision >= 0 { + if alt2 { + char_prefix_len(&t, precision as usize) + } else { + t.len().min(precision as usize) + } + } else { + t.len() + }; + body = t[..len].to_vec(); + width += utf8_extra(&body, alt2, width); + } + Kind::SqlEscape | Kind::SqlEscape2 | Kind::SqlEscape3 => { + let q = if inf.kind == Kind::SqlEscape3 { + b'"' + } else { + b'\'' + }; + let arg = args.text(); + let isnull = arg.is_none(); + let esc: Vec = match arg { + Some(t) => match t.iter().position(|&b| b == 0) { + Some(n) => t[..n].to_vec(), + None => t, + }, + None if inf.kind == Kind::SqlEscape2 => b"NULL".to_vec(), + None => b"(NULL)".to_vec(), + }; + // The precision counts input bytes (characters with `!`). + let mut k = precision; + let mut n = 0usize; + while k != 0 && n < esc.len() { + if alt2 && esc[n] & 0xc0 == 0xc0 { + while n + 1 < esc.len() && esc[n + 1] & 0xc0 == 0x80 { + n += 1; + } + } + n += 1; + k -= 1; + } + let need_quote = !isnull && inf.kind == Kind::SqlEscape2; + if need_quote { + body.push(q); + } + for &ch in &esc[..n] { + body.push(ch); + if ch == q { + body.push(ch); + } + } + if need_quote { + body.push(q); + } + width += utf8_extra(&body, alt2, width); + } + } + let length = body.len() as i64; + width -= length; + if width > 0 { + if !left { + out.pad(width, b' ')?; + } + out.push(&body)?; + if left { + out.pad(width, b' ')?; + } + } else { + out.push(&body)?; + } + } + Ok(out.finish()) +} + +/// With `!` and a positive width, the width counts characters: one more +/// for every UTF-8 continuation byte of the text. +fn utf8_extra(text: &[u8], alt2: bool, width: i64) -> i64 { + if alt2 && width > 0 { + text.iter().filter(|&&b| b & 0xc0 == 0x80).count() as i64 + } else { + 0 + } +} + +/// Bytes of the first `chars` characters of `t`. +fn char_prefix_len(t: &[u8], chars: usize) -> usize { + let mut z = 0usize; + let mut left = chars; + while left > 0 && z < t.len() { + let b = t[z]; + z += 1; + if b >= 0xc0 { + while z < t.len() && t[z] & 0xc0 == 0x80 { + z += 1; + } + } + left -= 1; + } + z +} + +/// The integer conversions (`d i u x X o p r`). +#[allow(clippy::too_many_arguments)] +fn integer( + mut longvalue: u64, + sign: u8, + inf: &Info, + mut precision: i64, + width: i64, + zeropad: bool, + mut alt: bool, + thousand: bool, +) -> SqawkResult> { + if longvalue == 0 { + alt = false; + } + let has_sign = i64::from(sign != 0); + if zeropad && precision < width - has_sign { + precision = width - has_sign; + } + if precision >= 37 { + temp_buf(precision + 10 + if thousand { precision / 3 } else { 0 })?; + } + // Built backwards, as SQLite builds it. + let mut rev: Vec = Vec::new(); + if inf.kind == Kind::Ordinal { + const ORD: &[u8; 8] = b"thstndrd"; + let mut x = (longvalue % 10) as usize; + if x >= 4 || (longvalue / 10) % 10 == 1 { + x = 0; + } + rev.push(ORD[x * 2 + 1]); + rev.push(ORD[x * 2]); + } + let digits: &[u8; 16] = if inf.upper { + b"0123456789ABCDEF" + } else { + b"0123456789abcdef" + }; + loop { + rev.push(digits[(longvalue % inf.base) as usize]); + longvalue /= inf.base; + if longvalue == 0 { + break; + } + } + while precision > rev.len() as i64 { + rev.push(b'0'); + } + let mut fwd: Vec = rev.into_iter().rev().collect(); + if thousand { + let mut grouped = Vec::with_capacity(fwd.len() + fwd.len() / 3); + let first = (fwd.len() - 1) % 3 + 1; + for (k, &b) in fwd.iter().enumerate() { + if k >= first && (k - first).is_multiple_of(3) { + grouped.push(b','); + } + grouped.push(b); + } + fwd = grouped; + } + let mut body = Vec::with_capacity(fwd.len() + 3); + if alt { + body.extend_from_slice(inf.alt_prefix); + } + if sign != 0 { + body.push(sign); + } + body.extend_from_slice(&fwd); + Ok(body) +} + +/// The real conversions (`f e E g G`). +#[allow(clippy::too_many_arguments)] +fn real( + v: f64, + kind: Kind, + upper: bool, + precision: i64, + width: i64, + flag_prefix: u8, + left: bool, + zeropad: bool, + alt: bool, + alt2: bool, + thousand: bool, +) -> SqawkResult> { + let mut precision = if precision < 0 { 6 } else { precision }; + precision = precision.min(FP_PRECISION_LIMIT); + // A `%g` precision of 0 rounds nothing (`%.0g` of 2.5 is `2`), as + // `%.0f` does. + let iround = match kind { + Kind::Float => -precision, + Kind::Generic => precision, + _ => precision + 1, + }; + let s = fp_decode(v, iround, if alt2 { 26 } else { 16 }); + let mut z: &[u8] = s.digits(); + let mut idp = s.idp; + if s.special == 2 { + return Ok(if zeropad { + b"null".to_vec() + } else { + b"NaN".to_vec() + }); + } + if s.special == 1 { + if zeropad { + z = b"9"; + idp = 1000; + } else { + let mut b = b"-Inf".to_vec(); + if s.sign == b'-' { + } else if flag_prefix != 0 { + b[0] = flag_prefix; + } else { + b.remove(0); + } + return Ok(b); + } + } + let prefix = if s.sign == b'-' { b'-' } else { flag_prefix }; + let exp = idp - 1; + let mut kind = kind; + let flag_rtz; + if kind == Kind::Generic && precision > 0 { + precision -= 1; + } + if kind == Kind::Generic { + flag_rtz = !alt; + if exp < -4 || exp > precision { + kind = Kind::Exp; + } else { + precision -= exp; + kind = Kind::Float; + } + } else { + flag_rtz = alt2; + } + let mut e2 = if kind == Kind::Exp { 0 } else { idp - 1 }; + let mut need = e2.max(0) + precision + width + 15; + if thousand && e2 > 0 { + need += (e2 + 2) / 3; + } + temp_buf(need)?; + let flag_dp = precision > 0 || alt || alt2; + let mut b: Vec = Vec::new(); + if prefix != 0 { + b.push(prefix); + } + let mut j = 0usize; + let digit = |j: &mut usize| { + if *j < z.len() { + *j += 1; + z[*j - 1] + } else { + b'0' + } + }; + if e2 < 0 { + b.push(b'0'); + } else { + while e2 >= 0 { + b.push(digit(&mut j)); + if thousand && e2 % 3 == 0 && e2 > 1 { + b.push(b','); + } + e2 -= 1; + } + } + if flag_dp { + b.push(b'.'); + } + e2 += 1; + while e2 < 0 && precision > 0 { + b.push(b'0'); + precision -= 1; + e2 += 1; + } + while precision > 0 { + b.push(digit(&mut j)); + precision -= 1; + } + if flag_rtz && flag_dp { + while b.last() == Some(&b'0') { + b.pop(); + } + if b.last() == Some(&b'.') { + if alt2 { + b.push(b'0'); + } else { + b.pop(); + } + } + } + if kind == Kind::Exp { + let mut exp = idp - 1; + b.push(if upper { b'E' } else { b'e' }); + if exp < 0 { + b.push(b'-'); + exp = -exp; + } else { + b.push(b'+'); + } + if exp >= 100 { + b.push(b'0' + (exp / 100) as u8); + exp %= 100; + } + b.push(b'0' + (exp / 10) as u8); + b.push(b'0' + (exp % 10) as u8); + } + let length = b.len() as i64; + if zeropad && !left && length < width { + let at = usize::from(prefix != 0); + let pad = (width - length) as usize; + b.splice(at..at, std::iter::repeat_n(b'0', pad)); + } + Ok(b) +} + +/// `printf` over a `&str` format, as text (bytes cut mid-character by a +/// `%s` precision become U+FFFD); `None` is SQL NULL. +pub(crate) fn format_str(fmt: &str, args: &[Register]) -> SqawkResult> { + Ok( + format(fmt.as_bytes(), args)?.map(|bytes| match String::from_utf8(bytes) { + Ok(s) => s, + Err(e) => String::from_utf8_lossy(e.as_bytes()).into_owned(), + }), + ) +} + +/// One real formatted by a single conversion (`"%.16g"`, `"%06.3f"`, ...). +pub(crate) fn real_str(fmt: &str, v: f64) -> String { + format_str(fmt, &[Register::Float(v)]) + .ok() + .flatten() + .unwrap_or_default() +} + +#[cfg(test)] +mod tests { + use super::*; + + fn p(fmt: &str, args: &[Register]) -> String { + format_str(fmt, args) + .unwrap() + .unwrap_or_else(|| "NULL".into()) + } + fn i(v: i64) -> Register { + Register::Integer(v) + } + fn f(v: f64) -> Register { + Register::Float(v) + } + fn s(v: &str) -> Register { + Register::String(v.into()) + } + + #[test] + fn integers() { + assert_eq!(p("%.3d", &[i(7)]), "007"); + assert_eq!(p("%u", &[i(-1)]), "18446744073709551615"); + assert_eq!(p("%,d", &[i(-1234567)]), "-1,234,567"); + assert_eq!(p("%#x|%#o|%#X", &[i(255), i(8), i(255)]), "0xff|010|0XFF"); + assert_eq!( + p("%r %r %r %r", &[i(1), i(2), i(13), i(-23)]), + "1st 2nd 13th -23rd" + ); + assert_eq!(p("%*d|%-*d|", &[i(4), i(7), i(-4), i(7)]), " 7|7 |"); + assert_eq!(p("%.*d", &[i(-3), i(7)]), "007"); + assert_eq!(p("%lld %ld", &[i(5), i(6)]), "5 6"); + assert_eq!(p("%d", &[s(" 1e3")]), "1"); + } + + #[test] + fn reals() { + assert_eq!(p("%.0f %.0f %.0f", &[f(0.5), f(1.5), f(2.5)]), "0 1 2"); + assert_eq!(p("%.20f", &[f(0.1)]), "0.10000000000000000000"); + assert_eq!(p("%!.20f", &[f(0.1)]), "0.1000000000000000055"); + assert_eq!( + p("%g|%#g|%G", &[f(0.00001), f(1.0), f(1e20)]), + "1e-05|1.00000|1E+20" + ); + assert_eq!(p("%e", &[f(0.0)]), "0.000000e+00"); + assert_eq!(p("%,.2f", &[f(1234567.891)]), "1,234,567.89"); + assert_eq!(p("%08.3f|%+.1e", &[f(-1.5), i(12345)]), "-001.500|+1.2e+04"); + assert_eq!( + p( + "%f|%5.1f|%010f", + &[f(f64::INFINITY), f(f64::NEG_INFINITY), f(1.0)] + ), + "Inf| -Inf|001.000000" + ); + assert_eq!(p("%!.15g", &[f(f64::INFINITY)]), "Inf"); + assert_eq!(p("%!0.15g", &[f(f64::INFINITY)]), "9.0e+999"); + } + + #[test] + fn strings() { + assert_eq!( + p( + "%q|%Q|%Q|%w", + &[s("it's"), s("a'b"), Register::Null, s("a\"b")] + ), + "it''s|'a''b'|NULL|a\"\"b" + ); + assert_eq!( + p("%5s|%-5s|%.2s", &[s("ab"), s("ab"), s("abc")]), + " ab|ab |ab" + ); + assert_eq!(p("%-4s|%!-4s|", &[s("é"), s("é")]), "é |é |"); + assert_eq!(p("%.3c|%c", &[s("xyz"), s("")]), "xxx|\0"); + assert_eq!(p("ab%5", &[]), "ab"); + assert_eq!(p("a%yb", &[]), "a"); + assert_eq!(p("50%", &[]), "50%"); + assert_eq!(p("%5%|%n|", &[]), " %||"); + assert_eq!(p("", &[]), "NULL"); + assert_eq!(p("%5", &[]), "NULL"); + assert_eq!(p("%s", &[s("")]), ""); + } +} diff --git a/src/vm/tests.rs b/src/vm/tests.rs index 69c77d1..add5234 100644 --- a/src/vm/tests.rs +++ b/src/vm/tests.rs @@ -708,181 +708,8 @@ mod bytecode_tests { other => panic!("Expected Integer type, got {:?}", other), } } - - /// Test transaction opcodes (Begin, Commit, Rollback) - #[test] - fn test_transaction_opcodes() { - // This test verifies that the transaction opcodes work correctly - let database = Database::new(); - - // Create a program that uses transaction opcodes - let instructions = vec![ - // Initialize VM - create_instruction( - OpCode::Init, - 0, - 1, - 0, - None, - Some("Initialize VM".to_string()), - ), - // Begin transaction - create_instruction( - OpCode::Begin, - 0, - 0, - 0, - None, - Some("Begin transaction".to_string()), - ), - // Load integer 42 into register 1 - create_instruction( - OpCode::Integer, - 42, - 1, - 0, - None, - Some("Load value 42".to_string()), - ), - // Commit transaction - create_instruction( - OpCode::Commit, - 0, - 0, - 0, - None, - Some("Commit transaction".to_string()), - ), - // Return result row with register 1 - create_instruction( - OpCode::ResultRow, - 1, - 1, - 0, - None, - Some("Return result".to_string()), - ), - // Halt execution - create_instruction( - OpCode::Halt, - 0, - 0, - 0, - None, - Some("Stop execution".to_string()), - ), - ]; - - // Execute the program - let result = execute_bytecode_program(instructions, &database) - .expect("Failed to execute program with Begin/Commit"); - - // Verify the result - assert!(result.is_some(), "Expected a result table"); - let table = result.unwrap(); - assert_eq!(table.row_count(), 1, "Expected 1 row"); - assert_eq!(table.column_count(), 1, "Expected 1 column"); - - // Check the value - let rows = table.rows(); - let first_row = &rows[0]; - match &first_row[0] { - Value::Integer(val) => assert_eq!(*val, 42, "Expected value 42, got {}", val), - other => panic!("Expected Integer type, got {:?}", other), - } - - // Test rollback - let instructions = vec![ - // Initialize VM - create_instruction( - OpCode::Init, - 0, - 1, - 0, - None, - Some("Initialize VM".to_string()), - ), - // Begin transaction - create_instruction( - OpCode::Begin, - 0, - 0, - 0, - None, - Some("Begin transaction".to_string()), - ), - // Load integer 42 into register 1 - create_instruction( - OpCode::Integer, - 42, - 1, - 0, - None, - Some("Load value in transaction".to_string()), - ), - // Rollback transaction - create_instruction( - OpCode::Rollback, - 0, - 0, - 0, - None, - Some("Rollback transaction".to_string()), - ), - // Load integer 99 into register 1 (after rollback) - create_instruction( - OpCode::Integer, - 99, - 1, - 0, - None, - Some("Load value after rollback".to_string()), - ), - // Return result row with register 1 - create_instruction( - OpCode::ResultRow, - 1, - 1, - 0, - None, - Some("Return result".to_string()), - ), - // Halt execution - create_instruction( - OpCode::Halt, - 0, - 0, - 0, - None, - Some("Stop execution".to_string()), - ), - ]; - - // Execute the program - let result = execute_bytecode_program(instructions, &database) - .expect("Failed to execute program with Begin/Rollback"); - - // Verify the result - assert!(result.is_some(), "Expected a result table"); - let table = result.unwrap(); - assert_eq!(table.row_count(), 1, "Expected 1 row"); - assert_eq!(table.column_count(), 1, "Expected 1 column"); - - // Check that we got the post-rollback value - let rows = table.rows(); - let first_row = &rows[0]; - match &first_row[0] { - Value::Integer(val) => { - assert_eq!(*val, 99, "Expected value 99 (after rollback), got {}", val) - } - other => panic!("Expected Integer type, got {:?}", other), - } - } } -// The compiler tests and other VM tests have been removed to simplify -// our implementation and focus on the transaction tests. - /// Test comparison opcodes and conditional jumps mod comparison_tests { use super::*; @@ -1292,108 +1119,6 @@ mod comparison_tests { } } - /// Test the IfNeg conditional jump opcode - #[test] - fn test_if_neg_opcode() { - let database = Database::new(); - - // Test cases: [(register_value, expected_result_after_jump)] - let test_cases = vec![ - (-5, 333), // -5 < 0 = true, should jump to 333 - (0, 444), // 0 = 0, should NOT jump, continue to 444 - (3, 444), // 3 > 0, should NOT jump, continue to 444 - ]; - - for (reg_value, expected) in test_cases { - let instructions = vec![ - // Initialize VM - create_instruction( - OpCode::Init, - 0, - 1, - 0, - None, - Some("Initialize VM".to_string()), - ), - // Load test value into register 1 - create_instruction( - OpCode::Integer, - reg_value, - 1, - 0, - None, - Some(format!("Load test value {}", reg_value)), - ), - // Jump to instruction 5 if reg1 is negative (< 0) - create_instruction( - OpCode::IfNeg, - 1, - 5, - 0, - None, - Some("Jump if negative".to_string()), - ), - // Not jumped path - load 444 into register 2 - create_instruction( - OpCode::Integer, - 444, - 2, - 0, - None, - Some("Load not-jumped value (444)".to_string()), - ), - // Jump to end - create_instruction(OpCode::Goto, 0, 6, 0, None, Some("Jump to end".to_string())), - // Jumped path - load 333 into register 2 - create_instruction( - OpCode::Integer, - 333, - 2, - 0, - None, - Some("Load jumped value (333)".to_string()), - ), - // Return result from register 2 - create_instruction( - OpCode::ResultRow, - 2, - 1, - 0, - None, - Some("Return result".to_string()), - ), - // Halt execution - create_instruction( - OpCode::Halt, - 0, - 0, - 0, - None, - Some("Stop execution".to_string()), - ), - ]; - - // Execute the program - let result = execute_bytecode_program(instructions, &database) - .expect("Failed to execute program with IfNeg"); - - // Verify the result - assert!(result.is_some(), "Expected a result table"); - let table = result.unwrap(); - assert_eq!(table.row_count(), 1, "Expected 1 row"); - - // Check the conditional jump outcome - match &table.rows()[0][0] { - Value::Integer(val) => assert_eq!( - *val, expected as i64, - "Expected {} for IfNeg test with value {}, got {}", - expected, reg_value, val - ), - other => panic!("Expected Integer type, got {:?}", other), - } - } - } - /// Test Column opcode with simple cursor operations #[test] fn test_column_opcode_basic() { diff --git a/src/vm/window.rs b/src/vm/window.rs index ce8a705..de0fbc1 100644 --- a/src/vm/window.rs +++ b/src/vm/window.rs @@ -6,12 +6,17 @@ //! into partitions, and within each partition into peer groups -- rows equal //! on every ORDER BY key, or the whole partition when there is no ORDER BY. //! Every function of the definition is then evaluated for each row and its -//! value stored in the function's result column. +//! value stored in the function's result column. An aggregate is evaluated +//! as SQLite evaluates it: rows are added to and removed from one running +//! state as the frame moves, in SQLite's order (see [`aggregate`]). use std::cmp::Ordering; +use std::collections::VecDeque; use super::agg::AggAccumulator; +use super::arith::text_of; use super::bytecode::{FrameBound, WindowDef, WindowFrame, WindowFunc, WindowFuncKind}; +use super::bytecode::{Register, AGG_GROUP_CONCAT, AGG_MAX, AGG_MIN, AGG_TYPE_MASK}; use super::engine::{compare_rows, compare_values}; use crate::error::{SqawkError, SqawkResult}; use crate::table::Value; @@ -222,30 +227,448 @@ fn evaluate(part: &[Vec], f: &WindowFunc, p: &Peers) -> SqawkResult { - // The accumulator covers rows [lo, hi). A frame that keeps its - // start and does not shrink (the default frame, ROWS UNBOUNDED - // PRECEDING, the whole partition) only steps the new rows; any - // other frame is accumulated afresh. - let mut acc = AggAccumulator::new(func_type); - let (mut lo, mut hi) = (0, 0); - for i in 0..n { - let (s, e) = frame(&f.frame, i, n, p); - if s != lo || e < hi { - acc = AggAccumulator::new(func_type); - lo = s; - hi = s; + WindowFuncKind::Agg(func_type) => return aggregate(part, f, func_type, p), + } + Ok(out) +} + +/// An aggregate's running state over a moving frame. +enum FrameAcc { + /// COUNT, SUM, AVG, TOTAL, and MIN / MAX over a frame that starts at + /// UNBOUNDED PRECEDING (which never removes a row). + Std(AggAccumulator), + /// MIN / MAX over a frame whose start moves: the frame's values, the + /// ones that can still become the result, oldest first (SQLite keeps an + /// ordered index of them). Of equal values the newest wins, as in + /// SQLite's index, whose last entry among equals is the newest. + MinMax { + min: bool, + live: VecDeque<(usize, Value)>, + next: usize, + removed: usize, + }, + GroupConcat(GroupConcat), +} + +impl FrameAcc { + fn new(func_type: i64, moving_start: bool) -> FrameAcc { + match func_type & AGG_TYPE_MASK { + t @ (AGG_MIN | AGG_MAX) if moving_start => FrameAcc::MinMax { + min: t == AGG_MIN, + live: VecDeque::new(), + next: 0, + removed: 0, + }, + AGG_GROUP_CONCAT => FrameAcc::GroupConcat(GroupConcat::default()), + _ => FrameAcc::Std(AggAccumulator::new(func_type)), + } + } + + fn step(&mut self, value: Option, sep: Option>) -> SqawkResult<()> { + match self { + FrameAcc::Std(acc) => acc.step(value, sep.flatten()), + FrameAcc::MinMax { + min, live, next, .. + } => { + let id = *next; + *next += 1; + let Some(v) = value.filter(|v| !matches!(v, Value::Null)) else { + return Ok(()); + }; + // A value no better than the new one can never be the + // result again: the new one is in every frame it is in. + let worse = if *min { + Ordering::Less + } else { + Ordering::Greater + }; + while live + .back() + .is_some_and(|(_, b)| compare_values(b, &v) != worse) + { + live.pop_back(); + } + live.push_back((id, v)); + Ok(()) + } + FrameAcc::GroupConcat(g) => { + g.step(value, sep); + Ok(()) + } + } + } + + /// Remove the oldest value still in the frame. + fn inverse(&mut self, value: Option) { + match self { + FrameAcc::Std(acc) => acc.inverse(value), + FrameAcc::MinMax { live, removed, .. } => { + if live.front().is_some_and(|(id, _)| id == removed) { + live.pop_front(); + } + *removed += 1; + } + FrameAcc::GroupConcat(g) => g.inverse(value), + } + } + + fn value(&self) -> SqawkResult { + match self { + FrameAcc::Std(acc) => acc.finalize(), + FrameAcc::MinMax { live, .. } => { + Ok(live.front().map_or(Value::Null, |(_, v)| v.clone())) + } + FrameAcc::GroupConcat(g) => Ok(g.value()), + } + } +} + +/// GROUP_CONCAT over a moving frame, as SQLite's `groupConcatStep` / +/// `groupConcatInverse` / `groupConcatValue` keep it: one byte string; a +/// row leaving the frame removes its value's bytes and the separator after +/// it from the front (the separator's length tracked per element once the +/// lengths vary). An emptied string starts over (the next value is a +/// first term, with no separator) and its value is NULL. +#[derive(Default)] +struct GroupConcat { + buf: Vec, + /// Bytes of `buf` already removed from the front. + head: usize, + /// Values presently concatenated. + n_accum: i64, + first_sep_len: usize, + /// The separators' lengths, oldest first, once they have varied. + sep_lens: Option>, + /// Something has been accumulated since the string was last empty. + active: bool, +} + +impl GroupConcat { + fn text(v: &Value) -> Vec { + text_of(&Register::from(v.clone())).into_bytes() + } + + /// `sep`: `None` for GROUP_CONCAT(x), `Some(separator)` with two + /// arguments. + fn step(&mut self, value: Option, sep: Option>) { + let Some(v) = value.filter(|v| !matches!(v, Value::Null)) else { + return; + }; + let first = !self.active; + self.active = true; + match sep { + None => { + if first { + self.first_sep_len = 1; + } else { + self.buf.push(b','); + } + } + Some(sep) => { + let sep = sep + .filter(|s| !matches!(s, Value::Null)) + .map(|s| Self::text(&s)); + let n_sep = sep.as_ref().map_or(0, Vec::len); + if first { + self.first_sep_len = n_sep; + } else { + if let Some(s) = &sep { + self.buf.extend_from_slice(s); + } + if n_sep != self.first_sep_len || self.sep_lens.is_some() { + let in_use = usize::try_from(self.n_accum - 1).unwrap_or(0); + let first_len = self.first_sep_len; + let lens = self.sep_lens.get_or_insert_with(|| { + std::iter::repeat_n(first_len, in_use).collect() + }); + lens.resize(in_use, first_len); + lens.push_back(n_sep); + } + } + } + } + self.n_accum += 1; + self.buf.extend_from_slice(&Self::text(&v)); + } + + fn inverse(&mut self, value: Option) { + let Some(v) = value.filter(|v| !matches!(v, Value::Null)) else { + return; + }; + let mut n = Self::text(&v).len(); + self.n_accum -= 1; + match self.sep_lens.as_mut() { + Some(lens) => { + if self.n_accum > 0 { + n += lens.pop_front().unwrap_or(0); + } + } + None => n += self.first_sep_len, + } + let len = self.buf.len() - self.head; + if n >= len { + self.buf.clear(); + self.head = 0; + } else { + self.head += n; + // Drop the dead prefix once it dominates, keeping removal + // amortized O(1) per byte. + if self.head > 4096 && self.head * 2 > self.buf.len() { + self.buf.drain(..self.head); + self.head = 0; + } + } + if self.buf.len() == self.head { + self.active = false; + self.sep_lens = None; + } + } + + fn value(&self) -> Value { + let bytes = &self.buf[self.head..]; + if bytes.is_empty() { + Value::Null + } else { + Value::String(String::from_utf8_lossy(bytes).into_owned().into()) + } + } +} + +/// A frame bound's offset as SQLite's countdown registers hold it. +fn offset(b: FrameBound) -> Option { + match b { + FrameBound::Preceding(x) | FrameBound::Following(x) => { + Some(i64::try_from(x).unwrap_or(i64::MAX)) + } + _ => None, + } +} + +/// SQLite's `OP_IfPos` countdown: `None` (no countdown) always runs; +/// otherwise a positive count is decremented and the operation skipped. +fn countdown(c: &mut Option) -> bool { + match c { + Some(x) if *x > 0 => { + *x -= 1; + false + } + _ => true, + } +} + +/// An aggregate window function over one partition, computed as SQLite's +/// `sqlite3WindowCodeStep` computes it: three cursors over the rows -- the +/// next row to add to the frame (step), the next to remove from it +/// (inverse) and the next to return -- advanced in SQLite's order for the +/// frame's shape, so each row's value comes from the same sequence of +/// steps and inverses as in SQLite (which decides rounding, and whether a +/// SUM is still an integer). Every row is added and removed at most once: +/// O(n) for any frame. A RANGE frame (bounds CURRENT ROW or UNBOUNDED) +/// moves by peer groups. +fn aggregate( + part: &[Vec], + f: &WindowFunc, + func_type: i64, + peers: &Peers, +) -> SqawkResult> { + let n = part.len(); + let mut out = vec![Value::Null; n]; + if n == 0 { + return Ok(out); + } + let fr = &f.frame; + // Units: rows for ROWS, peer groups for RANGE. + let units: Vec<(usize, usize)> = if fr.rows { + (0..n).map(|i| (i, i + 1)).collect() + } else { + let mut u = Vec::new(); + let mut i = 0; + while i < n { + u.push((i, peers.end[i])); + i = peers.end[i]; + } + u + }; + let nu = units.len(); + let unbounded_start = fr.start == FrameBound::UnboundedPreceding; + let is_following = |b: FrameBound| matches!(b, FrameBound::Following(_)); + let is_preceding = |b: FrameBound| matches!(b, FrameBound::Preceding(_)); + let end_unbounded = fr.end == FrameBound::UnboundedFollowing; + + let mut reg_start = offset(fr.start); + let mut reg_end = offset(fr.end); + // ROWS BETWEEN a FOLLOWING AND b FOLLOWING with b < a (or a PRECEDING + // AND b PRECEDING with b > a): every frame is empty. + if fr.rows { + if let (Some(s), Some(e)) = (reg_start, reg_end) { + let empty = (is_following(fr.start) && is_following(fr.end) && e < s) + || (is_preceding(fr.start) && is_preceding(fr.end) && e > s); + if empty { + let v = FrameAcc::new(func_type, !unbounded_start).value()?; + out.fill(v); + return Ok(out); + } + } + } + if is_following(fr.start) && is_following(fr.end) { + reg_start = reg_end.zip(reg_start).map(|(e, s)| e - s); + } + + let mut sim = Sim { + part, + f, + units: &units, + acc: FrameAcc::new(func_type, !unbounded_start), + removes: !unbounded_start, + out: &mut out, + end_c: 0, + cur_c: 0, + start_c: 0, + }; + + // The loop over input units: unit k has just been added to the table. + for _k in 1..nu { + if is_following(fr.start) { + sim.step()?; + if !end_unbounded { + if countdown(&mut reg_end) { + sim.ret()?; + } + if countdown(&mut reg_start) { + sim.inverse(); + } + } + } else if is_preceding(fr.end) { + if countdown(&mut reg_end) { + sim.step()?; + } + sim.ret()?; + if countdown(&mut reg_start) { + sim.inverse(); + } + } else { + sim.step()?; + if !end_unbounded { + if matches!(reg_end, Some(x) if x > 0) { + reg_end = reg_end.map(|x| x - 1); + } else { + sim.ret()?; + if countdown(&mut reg_start) { + sim.inverse(); + } + } + } + } + } + + // Flush: the partition is complete. + if is_preceding(fr.end) { + if countdown(&mut reg_end) { + sim.step()?; + } + sim.ret()?; + } else if is_following(fr.start) { + sim.step()?; + let mut tail = false; + loop { + if end_unbounded { + if countdown(&mut reg_start) { + sim.ret()?; + if sim.cur_c >= nu { + break; + } } - for row in &part[hi..e] { - acc.step( - f.args.first().map(|&c| row[c].clone()), - f.args.get(1).map(|&c| row[c].clone()), - )?; + sim.inverse(); + if sim.start_c >= nu { + tail = true; + break; } - hi = e; - out.push(acc.finalize()); + } else { + if countdown(&mut reg_end) { + sim.ret()?; + if sim.cur_c >= nu { + break; + } + } + if countdown(&mut reg_start) { + sim.inverse(); + if sim.start_c >= nu { + tail = true; + break; + } + } + } + } + if tail { + while sim.cur_c < nu { + sim.ret()?; + } + } + } else { + sim.step()?; + loop { + sim.ret()?; + if sim.cur_c >= nu { + break; + } + if countdown(&mut reg_start) { + sim.inverse(); } } } Ok(out) } + +/// The three cursors of [`aggregate`] over one partition's units (rows, or +/// peer groups). +struct Sim<'p> { + part: &'p [Vec], + f: &'p WindowFunc, + units: &'p [(usize, usize)], + acc: FrameAcc, + /// The frame's start moves (it is not UNBOUNDED PRECEDING). + removes: bool, + out: &'p mut [Value], + /// Next unit to add to the frame, to return, to remove from it. + end_c: usize, + cur_c: usize, + start_c: usize, +} + +impl Sim<'_> { + fn arg(&self, row: usize) -> Option { + self.f.args.first().map(|&c| self.part[row][c].clone()) + } + + fn step(&mut self) -> SqawkResult<()> { + if let Some(&(a, b)) = self.units.get(self.end_c) { + for row in a..b { + let sep = self.f.args.get(1).map(|&c| Some(self.part[row][c].clone())); + self.acc.step(self.arg(row), sep)?; + } + self.end_c += 1; + } + Ok(()) + } + + fn inverse(&mut self) { + if !self.removes { + return; + } + if let Some(&(a, b)) = self.units.get(self.start_c) { + for row in a..b { + self.acc.inverse(self.arg(row)); + } + self.start_c += 1; + } + } + + fn ret(&mut self) -> SqawkResult<()> { + if let Some(&(a, b)) = self.units.get(self.cur_c) { + let v = self.acc.value()?; + self.out[a..b].fill(v); + self.cur_c += 1; + } + Ok(()) + } +} diff --git a/tests/advanced/mod.rs b/tests/advanced/mod.rs index e3e272b..5df9abb 100644 --- a/tests/advanced/mod.rs +++ b/tests/advanced/mod.rs @@ -3,7 +3,7 @@ //! This module contains tests for complex features and scenarios that are not //! covered by the focused test modules. -use crate::helpers::create_temp_dir; +use crate::helpers::{assert_query_on, create_temp_dir}; use predicates::prelude::*; use std::fs; @@ -69,9 +69,68 @@ fn test_is_null_operator() -> Result<(), Box> { Ok(()) } -/// Skip testing complex WHERE expressions with logical operators for now -/// as there might be an issue with our test environment +/// Employees with NULL salaries and managers, for the three-valued WHERE tests. +const EMP: (&str, &str) = ( + "emp.csv", + "id,name,dept,salary,mgr\n\ + 1,Alice,Eng,100,\n\ + 2,Bob,Eng,80,1\n\ + 3,Carol,Sales,70,1\n\ + 4,Dan,Sales,,3\n\ + 5,Eve,Ops,90,\n", +); + +/// AND/OR/NOT and parentheses in WHERE: AND binds tighter than OR, NOT +/// tighter than both, and a NULL operand follows the SQL truth tables +/// (expected output computed with SQLite 3.45). #[test] fn test_complex_where_expressions() -> Result<(), Box> { - Ok(()) + // AND before OR. + assert_query_on( + &[EMP], + "SELECT id, name FROM emp WHERE dept = 'Eng' AND salary > 85 OR dept = 'Ops'", + "id,name\n1,Alice\n5,Eve", + )?; + // Parentheses override it. + assert_query_on( + &[EMP], + "SELECT id, name FROM emp WHERE dept = 'Eng' AND (salary > 85 OR dept = 'Ops')", + "id,name\n1,Alice", + )?; + // NOT (FALSE OR NULL) is NULL, so Dan (NULL salary) is excluded. + assert_query_on( + &[EMP], + "SELECT id, name FROM emp WHERE NOT (dept = 'Eng' OR salary < 75)", + "id,name\n5,Eve", + )?; + // NOT applies to the comparison only, then AND. + assert_query_on( + &[EMP], + "SELECT id, name FROM emp WHERE NOT dept = 'Sales' AND mgr IS NOT NULL", + "id,name\n2,Bob", + )?; + assert_query_on( + &[EMP], + "SELECT id, name FROM emp \ + WHERE (salary >= 80 AND NOT mgr IS NULL) OR (dept = 'Sales' AND salary IS NULL)", + "id,name\n2,Bob\n4,Dan", + )?; + // LIKE is case-insensitive: NOT (name LIKE 'c%') excludes Carol. + assert_query_on( + &[EMP], + "SELECT id, name FROM emp WHERE salary > 75 OR mgr = 1 AND NOT (name LIKE 'c%')", + "id,name\n1,Alice\n2,Bob\n5,Eve", + )?; + // The same operators as values: TRUE is 1, FALSE is 0. + assert_query_on( + &[EMP], + "SELECT id, salary > 75 AND mgr IS NULL, NOT (salary < 75) OR mgr = 3 \ + FROM emp ORDER BY id", + "id,salary > 75 AND mgr IS NULL,NOT (salary < 75) OR mgr = 3\n\ + 1,1,1\n\ + 2,0,1\n\ + 3,0,0\n\ + 4,0,1\n\ + 5,1,1", + ) } diff --git a/tests/aggregate/mod.rs b/tests/aggregate/mod.rs index fb102d4..831d68f 100644 --- a/tests/aggregate/mod.rs +++ b/tests/aggregate/mod.rs @@ -17,11 +17,13 @@ fn test_basic_aggregate_functions() -> Result<(), Box> { .arg("-v"); // employees.csv has 8 rows: - // COUNT(*) = 8, SUM(age) = 257, AVG(salary) = 67500, MIN(age) = 22, MAX(salary) = 90000 + // COUNT(*) = 8, SUM(age) = 257, AVG(salary) = 67500.0, MIN(age) = 22, MAX(salary) = 90000 cmd.assert() .success() - .stdout(predicate::str::contains("COUNT,SUM,AVG,MIN,MAX")) - .stdout(predicate::str::contains("8,257,67500,22,90000")); + .stdout(predicate::str::contains( + "COUNT(*),SUM(age),AVG(salary),MIN(age),MAX(salary)", + )) + .stdout(predicate::str::contains("8,257,67500.0,22,90000")); Ok(()) } @@ -60,11 +62,13 @@ fn test_aggregate_functions_with_filter() -> Result<(), Box 25: Alice(30), Charlie(35), David(40), Eve(28), Frank(32), Grace(45) = 6 rows - // COUNT = 6, SUM(salary) = 440000, AVG(age) = 35, MIN(salary) = 60000, MAX(age) = 45 + // COUNT = 6, SUM(salary) = 440000, AVG(age) = 35.0, MIN(salary) = 60000, MAX(age) = 45 cmd.assert() .success() - .stdout(predicate::str::contains("COUNT,SUM,AVG,MIN,MAX")) - .stdout(predicate::str::contains("6,440000,35,60000,45")); + .stdout(predicate::str::contains( + "COUNT(*),SUM(salary),AVG(age),MIN(salary),MAX(age)", + )) + .stdout(predicate::str::contains("6,440000,35.0,60000,45")); Ok(()) } @@ -152,12 +156,12 @@ mod sqlite_semantics { assert_query_on( &[EMP], "SELECT TOTAL(salary) AS t, TOTAL(NULL) AS n, typeof(TOTAL(salary)) AS ty FROM emp", - "t,n,ty\n300,0,real", + "t,n,ty\n300.0,0.0,real", )?; assert_query_on( &[EMP], "SELECT TOTAL(salary) AS t FROM emp WHERE id > 100", - "t\n0", + "t\n0.0", ) } @@ -196,7 +200,7 @@ mod sqlite_semantics { assert_query_on( &[ONE], &format!("SELECT AVG(x) AS a, TOTAL(x) AS t FROM {} z", BIG), - "a,t\n4611686018427388000,9223372036854776000", + "a,t\n4.61168601842739e+18,9.22337203685478e+18", ) } } diff --git a/tests/arithmetic/mod.rs b/tests/arithmetic/mod.rs index 9c39bf9..20d35fc 100644 --- a/tests/arithmetic/mod.rs +++ b/tests/arithmetic/mod.rs @@ -129,7 +129,7 @@ fn test_unary_minus() -> Result<(), Box> { // --------------------------------------------------------------------------- mod sqlite_semantics { - use crate::helpers::{assert_query_on, ONE}; + use crate::helpers::{assert_query_on, assert_query_on_fails, ONE}; type R = Result<(), Box>; @@ -144,10 +144,44 @@ mod sqlite_semantics { &[ONE], "SELECT 7/2 AS a, -7/2 AS b, 7%3 AS c, -7%3 AS d, 7%-3 AS e, 7.0/2 AS f, \ 5.5%2 AS h, -7.5%2 AS i FROM one", - "a,b,c,d,e,f,h,i\n3,-3,1,-1,1,3.5,1,-1", + "a,b,c,d,e,f,h,i\n3,-3,1,-1,1,3.5,1.0,-1.0", ) } + /// `& | << >>` and `~`: integers (text by its integer prefix, reals + /// truncated), shifts past 63 bits, negative shifts, NULL; they bind + /// below `+` and above `<`. + #[test] + fn bitwise_operators() -> R { + assert_query_on( + &[T], + "SELECT 12 & 10 AS a, 12 | 3 AS b, ~0 AS c, 1 << 63 AS d, 1 << 64 AS e, \ + -16 >> 2 AS f, -1 >> 64 AS g, 8 << -2 AS h, '12abc' & 7 AS i, 7.9 | 0 AS j, \ + ~a AS k, b << 1 AS l, 1 + 2 << 1 AS m, 1 | 2 < 3 AS n, ~~5 AS o FROM t WHERE k < 3", + "a,b,c,d,e,f,g,h,i,j,k,l,m,n,o\n8,15,-1,-9223372036854775808,0,-4,-1,2,4,7,-8,4,6,0,5\n\ + 8,15,-1,-9223372036854775808,0,-4,-1,2,4,7,6,,6,0,5", + ) + } + + /// Hex integer literals: 64-bit two's complement, 16 digits at most. + #[test] + fn hex_literals() -> R { + assert_query_on( + &[ONE], + "SELECT 0x1F AS a, 0XfF AS b, 0xFFFFFFFFFFFFFFFF AS c, 0x8000000000000000 AS d, \ + -0x10 AS e, 0x00000000000000000010 + 1 AS f, typeof(0x10) AS g, 0x10 FROM one", + "a,b,c,d,e,f,g,0x10\n31,255,-1,-9223372036854775808,-16,17,integer,16", + ) + } + + #[test] + fn hex_literal_errors() -> R { + assert_query_on_fails(&[ONE], "SELECT 0x10000000000000000", "hex literal too big")?; + assert_query_on_fails(&[ONE], "SELECT -0x8000000000000000", "hex literal too big")?; + assert_query_on_fails(&[ONE], "SELECT 0xZZ", "unrecognized token")?; + assert_query_on_fails(&[ONE], "SELECT X'41'", "blob literals are not supported") + } + #[test] fn division_by_zero_is_null() -> R { assert_query_on( @@ -190,7 +224,7 @@ mod sqlite_semantics { &[ONE], "SELECT '12abc' + 0 AS a, ' 5 ' + 1 AS b, 'abc' + 1 AS c, '1e3' + 0 AS d, \ typeof('1e3' + 0) AS e, '007' + 0 AS f, '3.9x' * 2 AS g FROM one", - "a,b,c,d,e,f,g\n12,6,1,1000,real,7,7.8", + "a,b,c,d,e,f,g\n12,6,1,1000.0,real,7,7.8", ) } @@ -201,7 +235,7 @@ mod sqlite_semantics { "SELECT k, a/2 AS h, a%3 AS m, a/b AS q, a%b AS r, b*2 AS d, typeof(a/2) AS ta, \ typeof(b*2) AS tb FROM t", "k,h,m,q,r,d,ta,tb\n\ - 1,3,1,2.8,1,5,integer,real\n\ + 1,3,1,2.8,1.0,5.0,integer,real\n\ 2,-3,-1,,,,integer,null\n\ 3,5,1,,,0,integer,integer", ) diff --git a/tests/basic/mod.rs b/tests/basic/mod.rs index 093957d..cecc8e2 100644 --- a/tests/basic/mod.rs +++ b/tests/basic/mod.rs @@ -134,7 +134,7 @@ fn test_custom_table_name() -> Result<(), Box> { fn test_invalid_sql() -> Result<(), Box> { let test_case = SqawkTestCase { sql: "INVALID SQL".to_string(), - expected_stderr: vec!["Failed to execute SQL".to_string()], + expected_stderr: vec!["sqawk: Expected: an SQL statement, found: INVALID".to_string()], should_succeed: false, ..Default::default() }; diff --git a/tests/cast/mod.rs b/tests/cast/mod.rs index 4f46b89..ba68f90 100644 --- a/tests/cast/mod.rs +++ b/tests/cast/mod.rs @@ -166,7 +166,7 @@ mod sqlite_semantics { &[ONE], "SELECT CAST('12abc' AS REAL) AS a, CAST('abc' AS REAL) AS b, \ CAST('1e3x' AS REAL) AS c, typeof(CAST(5 AS REAL)) AS d FROM one", - "a,b,c,d\n12,0,1000,real", + "a,b,c,d\n12.0,0.0,1000.0,real", ) } diff --git a/tests/common.rs b/tests/common.rs index af27008..6f3706b 100644 --- a/tests/common.rs +++ b/tests/common.rs @@ -13,7 +13,7 @@ pub fn run_repl_script( verbose: bool, ) -> String { // Build command args - let mut args = vec!["run", "--bin", "sqawk"]; + let mut args: Vec<&str> = Vec::new(); // Add input files args.extend(input_files); @@ -33,7 +33,7 @@ pub fn run_repl_script( args.push("--interactive"); // Start the sqawk process - let mut process = Command::new("cargo") + let mut process = Command::new(env!("CARGO_BIN_EXE_sqawk")) .args(&args) .stdin(Stdio::piped()) .stdout(Stdio::piped()) diff --git a/tests/comparison/mod.rs b/tests/comparison/mod.rs index 3eba3b4..cabe4b5 100644 --- a/tests/comparison/mod.rs +++ b/tests/comparison/mod.rs @@ -235,7 +235,7 @@ mod sqlite_semantics { assert_query( &[employees_csv()], "SELECT EMPLOYEES.NAME FROM EMPLOYEES WHERE ID = 2", - "NAME\nBob", + "name\nBob", ) } } diff --git a/tests/correlated/mod.rs b/tests/correlated/mod.rs index 60c5cf6..da30a20 100644 --- a/tests/correlated/mod.rs +++ b/tests/correlated/mod.rs @@ -1,7 +1,6 @@ //! Correlated subquery tests for sqawk VM //! //! This file contains tests for correlated subqueries. -//! Phase 4B: Correlated subquery support with runtime execution use crate::helpers::{create_custom_csv, create_temp_dir}; diff --git a/tests/date_functions/mod.rs b/tests/date_functions/mod.rs index 82573bf..2020fa3 100644 --- a/tests/date_functions/mod.rs +++ b/tests/date_functions/mod.rs @@ -89,7 +89,7 @@ fn test_current_date() -> Result<(), Box> { // --------------------------------------------------------------------------- mod sqlite_semantics { - use crate::helpers::{assert_query_on, assert_query_on_fails, ONE}; + use crate::helpers::{assert_query_on, ONE}; type R = Result<(), Box>; @@ -162,13 +162,107 @@ mod sqlite_semantics { ) } - /// sqawk works in UTC; 'localtime' is refused rather than ignored. + /// 'localtime' then 'utc' gives the moment back in any time zone; 'utc' + /// after an explicit zone does nothing more. #[test] - fn localtime_modifier_is_unsupported() -> R { - assert_query_on_fails( + fn localtime_utc_round_trip() -> R { + assert_query_on( + &[ONE], + "SELECT datetime('2024-01-15 10:30', 'localtime', 'utc') AS a, \ + datetime('2024-01-15 10:30+02:00', 'utc') AS b, \ + date('2024-07-04', 'localtime', 'utc') AS c FROM one", + "a,b,c\n2024-01-15 10:30:00,2024-01-15 08:30:00,2024-07-04", + ) + } + + /// 'localtime' and 'utc' follow the time zone of the TZ environment + /// variable, daylight saving time included (expected values from SQLite + /// 3.45 with the same TZ). + #[cfg(unix)] + #[test] + fn localtime_follows_tz() -> R { + let dir = crate::helpers::create_temp_dir()?; + let one = dir.path().join("one.csv"); + std::fs::write(&one, "id\n1\n")?; + let out = std::process::Command::new(env!("CARGO_BIN_EXE_sqawk")) + .env("TZ", "EST5EDT,M3.2.0,M11.1.0") + .arg("-s") + .arg( + "SELECT datetime('2024-07-01 10:00', 'localtime') AS a, \ + datetime('2024-01-01 10:00', 'localtime') AS b, \ + datetime('2024-03-10 02:30', 'utc') AS c, \ + datetime('1950-07-01 10:00', 'localtime') AS d FROM one", + ) + .arg(&one) + .output()?; + assert!( + out.status.success(), + "{}", + String::from_utf8_lossy(&out.stderr) + ); + assert_eq!( + String::from_utf8_lossy(&out.stdout), + "a,b,c,d\n2024-07-01 06:00:00,2024-01-01 05:00:00,2024-03-10 07:30:00,\ + 1950-07-01 06:00:00\n" + ); + Ok(()) + } + + /// Negative years, whitespace around the time, and the time string forms + /// SQLite accepts or rejects. + #[test] + fn time_string_forms() -> R { + assert_query_on( + &[ONE], + "SELECT date('-0001-12-31') AS a, date('2024-01-01 ') AS b, \ + datetime('2024-01-01 10:00') AS c, date(' 2024-01-01') AS d, \ + date('0000-01-01', '-1 day') AS e, time('24:00') AS f, \ + datetime('2024-01-01T10:00:00Z') AS g, date('20240101') AS h FROM one", + "a,b,c,d,e,f,g,h\n-0001-12-31,2024-01-01,2024-01-01 10:00:00,,-0001-12-31,\ + 24:00:00,2024-01-01 10:00:00,", + ) + } + + /// The modifiers SQLite 3.45 adds to the classic ones; 'ceiling' and + /// 'floor' arrived in 3.46 and are NULL, as in 3.45. + #[test] + fn newer_modifiers() -> R { + assert_query_on( + &[ONE], + "SELECT datetime('2024-01-01 10:00:00.123', 'subsec') AS a, \ + unixepoch('2024-01-01 10:00:00.5', 'subsec') AS b, \ + datetime(1700000000, 'auto') AS c, date(2460000.5, 'auto') AS d, \ + date(2460000.5, 'julianday') AS e, date(1700000000, 'julianday') AS f, \ + datetime('2024-01-01 10:00', '-01:30:15') AS g, \ + datetime('2024-01-31 10:00', '+0001-01-01 02:00') AS h, \ + date('2024-01-31', 'ceiling') AS i FROM one", + "a,b,c,d,e,f,g,h,i\n2024-01-01 10:00:00.123,1704103200.5,2023-11-14 22:13:20,\ + 2023-02-25,2023-02-25,,2024-01-01 08:29:45,2025-03-04 12:00:00,", + ) + } + + /// Every strftime conversion of SQLite 3.45; later ones (%G %V ...) are + /// NULL, as is a trailing '%'. + #[test] + fn strftime_conversions() -> R { + assert_query_on( + &[ONE], + "SELECT strftime('%d|%e|%f|%F|%H|%I|%j|%J|%k|%l|%m|%M|%p|%P|%R|%s|%S|%T|%u|%w|%W|%Y|%%', \ + '2024-07-07 15:06:07.089') AS a, strftime('%Y %F', '-0012-01-02') AS b, \ + strftime('%G', '2024-01-01') AS c, strftime('%Y%', '2024-01-01') AS d FROM one", + "a,b,c,d\n07| 7|07.089|2024-07-07|15|03|189|2460499.129248715|15| 3|07|06|PM|pm|\ + 15:06|1720364767|07|15:06:07|7|0|27|2024|%,-012 -012-01-02,,", + ) + } + + #[test] + fn timediff_values() -> R { + assert_query_on( &[ONE], - "SELECT date('now', 'localtime') FROM one", - "localtime", + "SELECT timediff('2024-03-01', '2023-01-15 12:00') AS a, \ + timediff('2023-01-15', '2024-03-01 12:00') AS b, \ + timediff('2024-01-01', NULL) AS c FROM one", + "a,b,c\n+0001-01-14 12:00:00.000,-0001-01-17 12:00:00.000,", ) } } diff --git a/tests/ddl/mod.rs b/tests/ddl/mod.rs index 3f9994c..f984343 100644 --- a/tests/ddl/mod.rs +++ b/tests/ddl/mod.rs @@ -43,7 +43,7 @@ fn test_drop_table_nonexistent() -> Result<(), Box> { cmd.assert() .failure() - .stderr(predicates::str::contains("not found")); + .stderr(predicates::str::contains("no such table: nonexistent")); Ok(()) } diff --git a/tests/defects/audit_2026_10.rs b/tests/defects/audit_2026_10.rs index e3889e1..a258e6c 100644 --- a/tests/defects/audit_2026_10.rs +++ b/tests/defects/audit_2026_10.rs @@ -337,9 +337,8 @@ fn non_canonical_numbers_display_verbatim() -> TestResult { ) } -/// The headerless auto-detection (file_handler.rs) guesses that a numeric -/// first line is data and names columns a,b. For .csv files the first line is -/// always the header. +/// A numeric first line is the header like any other: sqawk never guesses +/// that a file has no header row. #[test] fn csv_first_line_is_always_header() -> TestResult { assert_query_on( @@ -952,7 +951,7 @@ fn delete_with_join_is_error() -> TestResult { assert_query_fails( &[users_csv(), orders_csv()], "DELETE users FROM users JOIN orders ON users.id = orders.user_id", - "Unsupported", + "not supported", ) } @@ -1021,7 +1020,7 @@ fn insert_returning_is_error() -> TestResult { assert_query_fails( &[users_csv()], "INSERT INTO users VALUES (3, 'Zed', 'z@x') RETURNING id", - "Unsupported", + "not supported", ) } @@ -1031,7 +1030,7 @@ fn insert_on_conflict_is_error() -> TestResult { assert_query_fails( &[users_csv()], "INSERT INTO users VALUES (1, 'Zed', 'z@x') ON CONFLICT DO NOTHING", - "Unsupported", + "not supported", ) } @@ -1042,7 +1041,7 @@ fn insert_or_replace_is_error() -> TestResult { assert_query_fails( &[users_csv()], "INSERT OR REPLACE INTO users VALUES (1, 'Zed', 'z@x')", - "Unsupported", + "not supported", ) } @@ -1052,7 +1051,7 @@ fn update_from_is_error() -> TestResult { assert_query_fails( &[users_csv(), orders_csv()], "UPDATE users SET name = 'X' FROM orders WHERE users.id = 1", - "Unsupported", + "not supported", ) } @@ -1062,7 +1061,7 @@ fn delete_using_is_error() -> TestResult { assert_query_fails( &[users_csv(), orders_csv()], "DELETE FROM users USING orders WHERE users.id = 1", - "Unsupported", + "not supported", ) } @@ -1108,8 +1107,8 @@ fn percent_rank_function() -> TestResult { assert_query( &[employees_csv()], "SELECT id, PERCENT_RANK() OVER (ORDER BY department) AS f FROM employees", - "id,f\n1,0\n3,0\n6,0\n8,0.42857142857142855\n2,0.5714285714285714\n\ - 5,0.5714285714285714\n4,0.8571428571428571\n7,0.8571428571428571", + "id,f\n1,0.0\n3,0.0\n6,0.0\n8,0.428571428571429\n2,0.571428571428571\n\ + 5,0.571428571428571\n4,0.857142857142857\n7,0.857142857142857", ) } @@ -1331,14 +1330,14 @@ fn outer_join_aggregate_group_by() -> TestResult { } /// A scalar function over an aggregate (`ROUND(AVG(x), 1)`) is rejected with -/// GROUP BY ("must appear in GROUP BY"). Values follow sqawk's float -/// formatting (SQLite prints 70000.0). +/// GROUP BY ("must appear in GROUP BY"). Integral reals print as SQLite +/// prints them (70000.0). #[test] fn round_of_avg_with_group_by() -> TestResult { assert_query( &[employees_csv()], "SELECT department, ROUND(AVG(salary), 1) AS a FROM employees GROUP BY department", - "department,a\nEngineering,70000\nHR,45000\nMarketing,57500\nSales,85000", + "department,a\nEngineering,70000.0\nHR,45000.0\nMarketing,57500.0\nSales,85000.0", ) } diff --git a/tests/defects/code_review_2026_10.rs b/tests/defects/code_review_2026_10.rs index e88c1e2..ad0c0d9 100644 --- a/tests/defects/code_review_2026_10.rs +++ b/tests/defects/code_review_2026_10.rs @@ -168,6 +168,9 @@ fn printf_infinity_prints_inf() -> TestResult { // Finding 4: `#` comment lines are dropped by --write (csv_handler.rs:101) // --------------------------------------------------------------------------- +/// `#` lines are comments only under `--comments`. +const C: &[&str] = &["--comments"]; + /// A TSV with leading, interleaved and trailing comments. const TSV: &str = "# leading comment\n\ id\tname\tnote\n\ @@ -183,20 +186,14 @@ const TSV: &str = "# leading comment\n\ /// A no-op UPDATE must leave the file byte-identical, comments included. #[test] fn tsv_comments_survive_noop_update() -> TestResult { - assert_write_back("t.tsv", TSV, &[], "UPDATE t SET id = id WHERE id = 1", TSV) + assert_write_back("t.tsv", TSV, C, "UPDATE t SET id = id WHERE id = 1", TSV) } /// Comments keep their own line endings in a CRLF file. #[test] fn crlf_tsv_comments_survive_noop_update() -> TestResult { let crlf = "#c1\r\nid\tv\r\n#c2\r\n1\ta\r\n2\tb\r\n#end\r\n"; - assert_write_back( - "t.tsv", - crlf, - &[], - "UPDATE t SET id = id WHERE id = 1", - crlf, - ) + assert_write_back("t.tsv", crlf, C, "UPDATE t SET id = id WHERE id = 1", crlf) } /// A headerless colon-separated file (the /etc/passwd shape) with comments. @@ -211,7 +208,7 @@ fn headerless_colon_file_comments_survive_noop_update() -> TestResult { assert_write_back( "passwd.txt", passwd, - &["-F", ":"], + &["-F", ":", "--no-header", "--comments"], "UPDATE passwd SET c = c WHERE a = 'root'", passwd, ) @@ -223,7 +220,7 @@ fn comments_stay_put_on_update() -> TestResult { assert_write_back( "t.tsv", TSV, - &[], + C, "UPDATE t SET name = 'Zed' WHERE id = 2", &TSV.replace("Bob", "Zed"), ) @@ -236,7 +233,7 @@ fn comments_stay_put_on_delete() -> TestResult { assert_write_back( "t.tsv", TSV, - &[], + C, "DELETE FROM t WHERE id <> 2", "# leading comment\n\ id\tname\tnote\n\ @@ -255,7 +252,7 @@ fn insert_goes_before_trailing_comments() -> TestResult { assert_write_back( "t.tsv", TSV, - &[], + C, "INSERT INTO t VALUES (4, 'Dan', 'w')", "# leading comment\n\ id\tname\tnote\n\ @@ -272,19 +269,19 @@ fn insert_goes_before_trailing_comments() -> TestResult { } /// Edge cases, written with the fix: a `#` line inside a quoted multi-line -/// field is data, not a comment; blank lines are dropped (they always were); -/// a last comment line without a newline is reproduced as it is (the csv -/// crate reports it as an empty record, which used to load as a phantom -/// all-NULL row and be written back as a `\t` line). +/// field is data, not a comment; blank lines are kept; a last comment line +/// without a newline is reproduced as it is (the csv crate reports it as an +/// empty record, which used to load as a phantom all-NULL row and be written +/// back as a `\t` line). #[test] fn comment_edge_cases_survive_noop_update() -> TestResult { let input = "id\tnote\n1\t\"line one\n#not a comment\"\n\n# real comment\n2\tb\n#eof"; assert_write_back( "t.tsv", input, - &[], + C, "UPDATE t SET id = id WHERE id = 1", - &input.replace("\n\n", "\n"), + input, ) } @@ -298,8 +295,8 @@ fn comments_survive_deleting_every_row() -> TestResult { #another\n\ # trailing 1\n\ # trailing 2\n"; - assert_write_back("t.tsv", TSV, &[], "DELETE FROM t", comments_only)?; - assert_write_back("t.tsv", TSV, &[], "TRUNCATE TABLE t", comments_only) + assert_write_back("t.tsv", TSV, C, "DELETE FROM t", comments_only)?; + assert_write_back("t.tsv", TSV, C, "TRUNCATE TABLE t", comments_only) } // --------------------------------------------------------------------------- diff --git a/tests/defects/cte_2026_10.rs b/tests/defects/cte_2026_10.rs new file mode 100644 index 0000000..742652d --- /dev/null +++ b/tests/defects/cte_2026_10.rs @@ -0,0 +1,1281 @@ +//! Red list: common table expressions (`WITH [RECURSIVE] ...`). +//! +//! Every test in this module encodes **correct** behaviour for `WITH`, which +//! sqawk rejected outright. Expected outputs were computed by running the +//! same SQL over the same data in SQLite 3.45 (python3 `sqlite3`), unless +//! the doc comment says otherwise; errors are SQLite's messages. +//! +//! The rules covered: a CTE is materialized once, on first reference, and an +//! unused one is never evaluated; every CTE of a WITH list sees all its +//! siblings and itself; an inner WITH shadows an outer one; a column list +//! renames; only a one-part name can reference a CTE (`main.t` is the loaded +//! table); a CTE reference is qualified by its alias, else by the CTE name; +//! WITH works on SELECT, INSERT / UPDATE / DELETE, `INSERT ... WITH`, +//! `CREATE TABLE ... AS WITH` and inside any subquery; a CTE named like a +//! loaded table is read from EVERY clause position (never the loaded table); +//! and `WITH RECURSIVE` follows SQLite's queue semantics, checks and errors. +//! +//! Each test was `#[ignore]`d when written and un-ignored when WITH was +//! implemented (`src/vm/prepass.rs`); all of them now pass and guard +//! against regressions. Run the red list with: +//! cargo test --test mod defects -- --ignored + +use std::io::Write; +use std::process::{Command, Stdio}; + +use crate::helpers::{assert_query_on, assert_query_on_fails}; + +type TestResult = Result<(), Box>; + +/// `emp`: a small staff table (Ann's `mgr` is NULL). `tree`: a parent-child +/// table rooted at 1: 1 -> {2, 3}, 2 -> {4, 5}, 3 -> {6}, 4 -> {7}. +const FILES: &[(&str, &str)] = &[ + ( + "emp.csv", + "id,name,dept,mgr,salary\n1,Ann,eng,,100\n2,Bob,eng,1,80\n3,Cid,ops,1,70\n\ + 4,Dee,ops,3,60\n5,Eve,eng,2,90\n", + ), + ("tree.csv", "id,parent\n1,\n2,1\n3,1\n4,2\n5,2\n6,3\n7,4\n"), +]; + +/// Run the REPL over `people.csv` with `script` on stdin; return stdout. +fn repl(script: &str) -> Result> { + let mut child = Command::new(env!("CARGO_BIN_EXE_sqawk")) + .arg("-i") + .arg("tests/data/people.csv") + .stdin(Stdio::piped()) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .spawn()?; + child + .stdin + .as_mut() + .ok_or("no stdin")? + .write_all(script.as_bytes())?; + let out = child.wait_with_output()?; + Ok(String::from_utf8_lossy(&out.stdout).replace("\r\n", "\n")) +} + +/// A CTE body is held to the parser's limits like any query: a 1001-deep +/// expression in it is too deep. (sqawk's own limit, SQLite's +/// `SQLITE_MAX_EXPR_DEPTH`.) +#[test] +fn cte_body_depth_limit() -> TestResult { + let chain = vec!["1"; 1001].join("+"); + assert_query_on_fails( + FILES, + &format!("WITH c AS (SELECT {chain} AS v) SELECT v FROM c"), + "expression tree is too large (maximum depth 1000)", + ) +} + +/// `WITH ... DELETE` reports its row count like a plain DELETE, and the CTE +/// leaves nothing registered: `.tables` lists only the loaded table, and the +/// CTE name is free for the next statement. (sqawk REPL behaviour.) +#[test] +fn with_dml_counts_rows_and_leaks_nothing() -> TestResult { + let stdout = repl( + ".changes on\n\ + WITH r AS (SELECT id FROM people WHERE age > 30) \ + DELETE FROM people WHERE id IN (SELECT id FROM r);\n\ + WITH r AS (SELECT 1 AS id) SELECT id FROM r;\n\ + .tables\n\ + .exit\n", + )?; + assert!( + stdout.contains("2 rows affected"), + "row count missing, got:\n{stdout}" + ); + assert!( + !stdout.contains("sqawk_"), + "an internal table leaked, got:\n{stdout}" + ); + assert!( + !stdout.lines().any(|l| l.trim() == "r"), + ".tables lists the CTE, got:\n{stdout}" + ); + Ok(()) +} + +/// A statement whose CTE fails leaves nothing registered either. +/// (sqawk REPL behaviour.) +#[test] +fn failed_cte_statement_leaks_nothing() -> TestResult { + let stdout = repl( + "WITH a AS (SELECT 1 AS v), b AS (SELECT nosuch FROM a) SELECT * FROM a, b;\n\ + WITH a AS (SELECT 2 AS v) SELECT v FROM a;\n\ + .tables\n\ + .exit\n", + )?; + assert!( + stdout.contains("v\n2\n"), + "the second statement failed, got:\n{stdout}" + ); + assert!( + !stdout.contains("sqawk_"), + "an internal table leaked, got:\n{stdout}" + ); + Ok(()) +} + +/// A recursive counter to 1000 terminates through its WHERE. +/// (Expected computed: 1000 rows summing to 500500.) +#[test] +fn recursive_counter_to_1000() -> TestResult { + assert_query_on( + FILES, + "WITH RECURSIVE c(n) AS (SELECT 1 UNION ALL SELECT n + 1 FROM c WHERE n < 1000) \ + SELECT COUNT(*), SUM(n), MAX(n) FROM c", + "COUNT(*),SUM(n),MAX(n)\n1000,500500,1000", + ) +} + +/// A CTE named like a loaded table, read from a plain FROM. +#[test] +fn shadow_in_from() -> TestResult { + assert_query_on( + FILES, + "WITH tree AS (SELECT 42 AS id) SELECT id FROM tree", + "id\n42", + ) +} + +/// A CTE reference inside a recursive CTE's setup arm reads the sibling. +#[test] +fn recursive_setup_reads_sibling() -> TestResult { + assert_query_on( + FILES, + "WITH s AS (SELECT 5 AS start), \ + c(n) AS (SELECT start FROM s UNION ALL SELECT n + 1 FROM c WHERE n < 7) \ + SELECT n FROM c", + "n\n5\n6\n7", + ) +} + +// --------------------------------------------------------------------------- +// Plain CTEs +// --------------------------------------------------------------------------- + +/// A CTE is a named query in scope for the statement. +#[test] +fn plain_cte() -> TestResult { + assert_query_on( + FILES, + "WITH h AS (SELECT name, salary FROM emp WHERE salary > 75) SELECT name FROM h ORDER BY name", + "name\n\ + Ann\n\ + Bob\n\ + Eve", + ) +} + +/// A column list renames the CTE's columns. +#[test] +fn column_list_renames() -> TestResult { + assert_query_on( + FILES, + "WITH h(n, s) AS (SELECT name, salary FROM emp) SELECT n, s FROM h WHERE s < 75 ORDER BY n", + "n,s\n\ + Cid,70\n\ + Dee,60", + ) +} + +/// `SELECT *` from a CTE with a column list uses those names. +#[test] +fn star_uses_cte_column_names() -> TestResult { + assert_query_on( + FILES, + "WITH h(n, s) AS (SELECT name, salary FROM emp WHERE id = 1) SELECT * FROM h", + "n,s\n\ + Ann,100", + ) +} + +/// A CTE reads one declared before it. +#[test] +fn chained_ctes() -> TestResult { + assert_query_on( + FILES, + "WITH a AS (SELECT 1 AS n), b AS (SELECT n + 1 AS n FROM a) SELECT n FROM b", + "n\n\ + 2", + ) +} + +/// Every CTE of a WITH list sees all its siblings, including those declared +/// after it. +#[test] +fn forward_reference_to_sibling() -> TestResult { + assert_query_on( + FILES, + "WITH a AS (SELECT * FROM b), b AS (SELECT 1 AS z) SELECT * FROM a", + "z\n\ + 1", + ) +} + +/// An unused CTE is never evaluated, even when its body would fail. +#[test] +fn unused_cte_is_never_evaluated() -> TestResult { + assert_query_on( + FILES, + "WITH x AS (SELECT * FROM nosuch) SELECT 1", + "1\n\ + 1", + ) +} + +/// An inner WITH shadows an outer CTE of the same name. +#[test] +fn inner_with_shadows_outer() -> TestResult { + assert_query_on( + FILES, + "WITH c AS (SELECT 1 AS v) SELECT (WITH c AS (SELECT 2 AS v) SELECT v FROM c) AS inner_v, v FROM c", + "inner_v,v\n\ + 2,1", + ) +} + +/// A CTE body resolves names where it is defined, not where it is referenced. +#[test] +fn cte_body_resolves_in_its_own_scope() -> TestResult { + assert_query_on( + FILES, + "WITH a AS (SELECT v FROM b), b AS (SELECT 1 AS v) SELECT * FROM (WITH b AS (SELECT 2 AS v) SELECT v FROM a) d", + "v\n\ + 1", + ) +} + +/// One CTE may be referenced twice in a statement. +#[test] +fn cte_referenced_twice() -> TestResult { + assert_query_on( + FILES, + "WITH h AS (SELECT id FROM emp WHERE id < 4) SELECT x.id, y.id FROM h x JOIN h y ON x.id < y.id ORDER BY x.id, y.id", + "id,id\n\ + 1,2\n\ + 1,3\n\ + 2,3", + ) +} + +/// An unaliased CTE reference is qualified by the CTE's name. +#[test] +fn cte_name_qualifies_columns() -> TestResult { + assert_query_on( + FILES, + "WITH c AS (SELECT 1 AS v) SELECT c.v FROM c", + "v\n\ + 1", + ) +} + +/// An aliased CTE reference is qualified by its alias. +#[test] +fn cte_alias_qualifies_columns() -> TestResult { + assert_query_on( + FILES, + "WITH c AS (SELECT 1 AS v) SELECT x.v FROM c AS x", + "v\n\ + 1", + ) +} + +/// Once aliased, the CTE's own name no longer qualifies its columns. +#[test] +fn cte_alias_hides_cte_name() -> TestResult { + assert_query_on_fails( + FILES, + "WITH c AS (SELECT 1 AS v) SELECT c.v FROM c AS x", + "no such column: c.v", + ) +} + +/// Errors name the CTE as written, never an internal table. +#[test] +fn unknown_column_names_the_cte() -> TestResult { + assert_query_on_fails( + FILES, + "WITH c AS (SELECT 1 AS v) SELECT c.nosuch FROM c", + "no such column: c.nosuch", + ) +} + +/// A CTE shadows a loaded table of the same name. +#[test] +fn cte_shadows_loaded_table() -> TestResult { + assert_query_on( + FILES, + "WITH emp AS (SELECT 9 AS id) SELECT id FROM emp", + "id\n\ + 9", + ) +} + +/// `main.emp` names the loaded table, never a CTE. +#[test] +fn main_schema_reads_the_real_table() -> TestResult { + assert_query_on( + FILES, + "WITH emp AS (SELECT 9 AS id) SELECT id FROM main.emp ORDER BY id", + "id\n\ + 1\n\ + 2\n\ + 3\n\ + 4\n\ + 5", + ) +} + +/// CTE names match case-insensitively. +#[test] +fn cte_names_match_case_insensitively() -> TestResult { + assert_query_on( + FILES, + "WITH Cte AS (SELECT 1 AS v) SELECT v FROM CTE", + "v\n\ + 1", + ) +} + +/// Duplicate column names in a CTE are made unique as in a derived table. +/// (sqawk's derived-table naming; SQLite's own `a:1` is internal.) +#[test] +fn duplicate_result_columns_are_deduplicated() -> TestResult { + assert_query_on( + FILES, + "WITH c AS (SELECT 1 AS a, 2 AS a) SELECT * FROM c", + "a,a:1\n\ + 1,2", + ) +} + +/// A compound inside a CTE body evaluates left to right like any compound. +#[test] +fn cte_body_compound_is_left_associative() -> TestResult { + assert_query_on( + FILES, + "WITH u AS (SELECT 1 AS v UNION SELECT 2 INTERSECT SELECT 2) SELECT * FROM u", + "v\n\ + 2", + ) +} + +/// `main.t.c` inside a CTE body is `t.c`. +#[test] +fn cte_body_strips_main_schema() -> TestResult { + assert_query_on( + FILES, + "WITH c AS (SELECT main.emp.id AS i FROM emp WHERE id = 1) SELECT i FROM c", + "i\n\ + 1", + ) +} + +/// `AS MATERIALIZED` and `AS NOT MATERIALIZED` are accepted hints. +#[test] +fn materialized_hint_is_accepted() -> TestResult { + assert_query_on( + FILES, + "WITH c AS MATERIALIZED (SELECT 1 AS v), d AS NOT MATERIALIZED (SELECT 2 AS w) SELECT v, w FROM c, d", + "v,w\n\ + 1,2", + ) +} + +/// WITH inside a scalar subquery. +#[test] +fn cte_in_scalar_subquery() -> TestResult { + assert_query_on( + FILES, + "SELECT name FROM emp WHERE salary = (WITH m AS (SELECT MAX(salary) AS s FROM emp) SELECT s FROM m)", + "name\n\ + Ann", + ) +} + +/// WITH inside a derived table. +#[test] +fn cte_in_derived_table() -> TestResult { + assert_query_on( + FILES, + "SELECT v FROM (WITH c AS (SELECT 5 AS v) SELECT v FROM c) t", + "v\n\ + 5", + ) +} + +/// WITH inside an IN subquery. +#[test] +fn cte_in_in_subquery() -> TestResult { + assert_query_on( + FILES, + "SELECT id FROM emp WHERE id IN (WITH h AS (SELECT 1 AS x UNION ALL SELECT 3) SELECT x FROM h) ORDER BY id", + "id\n\ + 1\n\ + 3", + ) +} + +/// A CTE referenced from a correlated subquery of the main query. +#[test] +fn cte_in_correlated_subquery() -> TestResult { + assert_query_on( + FILES, + "WITH m AS (SELECT dept, MAX(salary) AS mx FROM emp GROUP BY dept) SELECT name FROM emp WHERE salary = (SELECT mx FROM m WHERE m.dept = emp.dept) ORDER BY name", + "name\n\ + Ann\n\ + Cid", + ) +} + +/// A CTE body that refers to an outer query's column cannot be computed up +/// front; like a correlated derived table it is a clean error. (SQLite +/// evaluates it per outer row; a documented difference.) +#[test] +fn correlated_cte_body_is_an_error() -> TestResult { + assert_query_on_fails( + FILES, + "SELECT (WITH c AS (SELECT emp.id AS v) SELECT v FROM c) FROM emp", + "no such column: emp.id", + ) +} + +// --------------------------------------------------------------------------- +// Errors +// --------------------------------------------------------------------------- + +/// A non-recursive CTE that reads its own name is a circular reference. +#[test] +fn self_reference_is_circular() -> TestResult { + assert_query_on_fails( + FILES, + "WITH emp AS (SELECT * FROM emp) SELECT * FROM emp", + "circular reference: emp", + ) +} + +/// Two CTEs that read each other are a circular reference. +#[test] +fn mutual_reference_is_circular() -> TestResult { + assert_query_on_fails( + FILES, + "WITH a AS (SELECT * FROM b), b AS (SELECT * FROM a) SELECT * FROM a", + "circular reference: a", + ) +} + +/// Two CTEs of one WITH may not share a name. +#[test] +fn duplicate_cte_name() -> TestResult { + assert_query_on_fails( + FILES, + "WITH c AS (SELECT 1), C AS (SELECT 2) SELECT 1", + "duplicate WITH table name: C", + ) +} + +/// A column list must match the body's column count. +#[test] +fn column_list_count_mismatch() -> TestResult { + assert_query_on_fails( + FILES, + "WITH c(a, b) AS (SELECT 1) SELECT * FROM c", + "table c has 1 values for 2 columns", + ) +} + +/// WITH on a parenthesized compound operand stays rejected (a syntax error +/// in SQLite). +#[test] +fn compound_operand_with_is_rejected() -> TestResult { + assert_query_on_fails( + FILES, + "SELECT 1 UNION (WITH x AS (SELECT 2) SELECT * FROM x)", + "ORDER BY, LIMIT or WITH inside a compound SELECT operand is not supported", + ) +} + +// --------------------------------------------------------------------------- +// Statement forms +// --------------------------------------------------------------------------- + +/// `WITH ... INSERT`. +#[test] +fn with_insert() -> TestResult { + assert_query_on( + FILES, + "WITH n AS (SELECT 6 AS id, 'Fay' AS name) INSERT INTO emp (id, name) SELECT id, name FROM n; SELECT id, name FROM emp WHERE id > 4 ORDER BY id", + "id,name\n\ + 5,Eve\n\ + 6,Fay", + ) +} + +/// `INSERT INTO t WITH ... SELECT`. +#[test] +fn insert_with_select() -> TestResult { + assert_query_on( + FILES, + "INSERT INTO emp (id, name) WITH n AS (SELECT 7 AS id, 'Gus' AS name) SELECT * FROM n; SELECT id, name FROM emp WHERE id > 4 ORDER BY id", + "id,name\n\ + 5,Eve\n\ + 7,Gus", + ) +} + +/// `WITH ... UPDATE`. +#[test] +fn with_update() -> TestResult { + assert_query_on( + FILES, + "WITH r AS (SELECT id FROM emp WHERE dept = 'ops') UPDATE emp SET salary = 0 WHERE id IN (SELECT id FROM r); SELECT id, salary FROM emp ORDER BY id", + "id,salary\n\ + 1,100\n\ + 2,80\n\ + 3,0\n\ + 4,0\n\ + 5,90", + ) +} + +/// `WITH ... DELETE`. +#[test] +fn with_delete() -> TestResult { + assert_query_on( + FILES, + "WITH r AS (SELECT id FROM emp WHERE salary < 75) DELETE FROM emp WHERE id IN (SELECT id FROM r); SELECT id FROM emp ORDER BY id", + "id\n\ + 1\n\ + 2\n\ + 5", + ) +} + +/// `CREATE TABLE ... AS WITH ...`. +#[test] +fn create_table_as_with() -> TestResult { + assert_query_on( + FILES, + "CREATE TABLE hi AS WITH r AS (SELECT name FROM emp WHERE salary >= 90) SELECT name FROM r; SELECT * FROM hi ORDER BY name", + "name\n\ + Ann\n\ + Eve", + ) +} + +/// A DML target names a loaded table even when a CTE shares its name. +#[test] +fn dml_target_is_always_a_real_table() -> TestResult { + assert_query_on( + FILES, + "WITH emp AS (SELECT 1 AS id) DELETE FROM emp WHERE id = 1; SELECT id FROM emp ORDER BY id", + "id\n\ + 2\n\ + 3\n\ + 4\n\ + 5", + ) +} + +/// A DML target that only a CTE defines is no such table. +#[test] +fn dml_target_never_a_cte() -> TestResult { + assert_query_on_fails( + FILES, + "WITH c AS (SELECT 1 AS x) UPDATE c SET x = 2", + "no such table: c", + ) +} + +// --------------------------------------------------------------------------- +// Every clause position reads the CTE, never the same-named loaded table +// --------------------------------------------------------------------------- + +/// A CTE named like a loaded table, read from a JOIN. +#[test] +fn shadow_in_join() -> TestResult { + assert_query_on( + FILES, + "WITH emp AS (SELECT 2 AS id) SELECT t.id FROM tree t JOIN emp ON emp.id = t.id", + "id\n\ + 2", + ) +} + +/// ... from a subquery in JOIN ON. +#[test] +fn shadow_in_join_on_subquery() -> TestResult { + assert_query_on( + FILES, + "WITH emp AS (SELECT 2 AS id) SELECT t.id FROM tree t JOIN tree u ON u.id = t.id AND t.id IN (SELECT id FROM emp)", + "id\n\ + 2", + ) +} + +/// ... from a subquery in WHERE. +#[test] +fn shadow_in_where_subquery() -> TestResult { + assert_query_on( + FILES, + "WITH emp AS (SELECT 2 AS id) SELECT id FROM tree WHERE id IN (SELECT id FROM emp)", + "id\n\ + 2", + ) +} + +/// ... from a subquery in GROUP BY. +#[test] +fn shadow_in_group_by_subquery() -> TestResult { + assert_query_on( + FILES, + "WITH emp AS (SELECT 2 AS id) SELECT COUNT(*) FROM tree GROUP BY id > (SELECT id FROM emp) ORDER BY 1", + "COUNT(*)\n\ + 2\n\ + 5", + ) +} + +/// ... from a subquery in HAVING. +#[test] +fn shadow_in_having_subquery() -> TestResult { + assert_query_on( + FILES, + "WITH emp AS (SELECT 100 AS id) SELECT COUNT(*) FROM tree HAVING COUNT(*) > (SELECT id FROM emp)", + "COUNT(*)", + ) +} + +/// ... from a subquery in ORDER BY. +#[test] +fn shadow_in_order_by_subquery() -> TestResult { + assert_query_on( + FILES, + "WITH emp AS (SELECT -1 AS id) SELECT id FROM tree ORDER BY id * (SELECT id FROM emp) LIMIT 3", + "id\n\ + 7\n\ + 6\n\ + 5", + ) +} + +/// ... from a subquery in a window's PARTITION BY. +#[test] +fn shadow_in_window_partition() -> TestResult { + assert_query_on( + FILES, + "WITH emp AS (SELECT 5 AS id) SELECT id, COUNT(*) OVER (PARTITION BY id > (SELECT id FROM emp)) AS k FROM tree ORDER BY id", + "id,k\n\ + 1,5\n\ + 2,5\n\ + 3,5\n\ + 4,5\n\ + 5,5\n\ + 6,2\n\ + 7,2", + ) +} + +/// ... from a subquery in a window's ORDER BY. +#[test] +fn shadow_in_window_order() -> TestResult { + assert_query_on( + FILES, + "WITH emp AS (SELECT -1 AS id) SELECT id, ROW_NUMBER() OVER (ORDER BY id * (SELECT id FROM emp)) AS r FROM tree ORDER BY id", + "id,r\n\ + 1,7\n\ + 2,6\n\ + 3,5\n\ + 4,4\n\ + 5,3\n\ + 6,2\n\ + 7,1", + ) +} + +/// ... from a subquery as a function argument. +#[test] +fn shadow_in_function_argument() -> TestResult { + assert_query_on( + FILES, + "WITH emp AS (SELECT 100 AS id) SELECT COALESCE((SELECT id FROM emp), 0) AS v, ABS((SELECT -id FROM emp)) AS w", + "v,w\n\ + 100,100", + ) +} + +/// ... from a subquery in an aggregate's FILTER. +#[test] +fn shadow_in_aggregate_filter() -> TestResult { + assert_query_on( + FILES, + "WITH emp AS (SELECT 2 AS id) SELECT COUNT(*) FILTER (WHERE id IN (SELECT id FROM emp)) AS n FROM tree", + "n\n\ + 1", + ) +} + +/// ... from a subquery in CASE. +#[test] +fn shadow_in_case() -> TestResult { + assert_query_on( + FILES, + "WITH emp AS (SELECT 100 AS id) SELECT CASE WHEN (SELECT id FROM emp) = 100 THEN 'cte' ELSE 'real' END AS which", + "which\n\ + cte", + ) +} + +/// ... from a subquery in LIMIT. +#[test] +fn shadow_in_limit() -> TestResult { + assert_query_on( + FILES, + "WITH emp AS (SELECT 2 AS id) SELECT id FROM tree ORDER BY id LIMIT (SELECT id FROM emp)", + "id\n\ + 1\n\ + 2", + ) +} + +/// ... from subqueries in BETWEEN and an IN list. +#[test] +fn shadow_in_between_and_in_list() -> TestResult { + assert_query_on( + FILES, + "WITH emp AS (SELECT 6 AS id) SELECT id FROM tree WHERE id BETWEEN (SELECT id FROM emp) AND 9 OR id IN (1, (SELECT id - 1 FROM emp)) ORDER BY id", + "id\n\ + 1\n\ + 5\n\ + 6\n\ + 7", + ) +} + +/// ... from EXISTS. +#[test] +fn shadow_in_exists() -> TestResult { + assert_query_on( + FILES, + "WITH emp AS (SELECT 100 AS id) SELECT COUNT(*) FROM tree WHERE EXISTS (SELECT 1 FROM emp WHERE emp.id = tree.id)", + "COUNT(*)\n\ + 0", + ) +} + +/// ... from a compound's second arm. +#[test] +fn shadow_in_compound_arm() -> TestResult { + assert_query_on( + FILES, + "WITH emp AS (SELECT 100 AS id) SELECT id FROM tree WHERE id = 1 UNION ALL SELECT id FROM emp", + "id\n\ + 1\n\ + 100", + ) +} + +/// ... from a derived table. +#[test] +fn shadow_in_derived_table() -> TestResult { + assert_query_on( + FILES, + "WITH emp AS (SELECT 2 AS id) SELECT * FROM (SELECT id FROM emp) d", + "id\n\ + 2", + ) +} + +/// ... from a sibling CTE's body. +#[test] +fn shadow_in_sibling_cte() -> TestResult { + assert_query_on( + FILES, + "WITH emp AS (SELECT 2 AS id), d AS (SELECT id FROM emp) SELECT * FROM d", + "id\n\ + 2", + ) +} + +/// ... from a subquery in UPDATE SET. +#[test] +fn shadow_in_update_set() -> TestResult { + assert_query_on( + FILES, + "WITH emp AS (SELECT 100 AS id) UPDATE tree SET parent = (SELECT id FROM emp) WHERE id = 1; SELECT parent FROM tree WHERE id = 1", + "parent\n\ + 100", + ) +} + +/// ... from an INSERT's source. +#[test] +fn shadow_in_insert_source() -> TestResult { + assert_query_on( + FILES, + "WITH emp AS (SELECT 100 AS id) INSERT INTO tree (id) SELECT id FROM emp; SELECT COUNT(*) FROM tree WHERE id = 100", + "COUNT(*)\n\ + 1", + ) +} + +/// ... from CREATE TABLE AS. +#[test] +fn shadow_in_ctas() -> TestResult { + assert_query_on( + FILES, + "CREATE TABLE k AS WITH emp AS (SELECT 100 AS id) SELECT id FROM emp; SELECT * FROM k", + "id\n\ + 100", + ) +} + +/// Outside the CTE's scope the same name is the loaded table. +#[test] +fn loaded_table_outside_cte_scope() -> TestResult { + assert_query_on( + FILES, + "SELECT (SELECT COUNT(*) FROM emp) AS a, (WITH emp AS (SELECT 1 AS id) SELECT COUNT(*) FROM emp) AS b", + "a,b\n\ + 5,1", + ) +} + +/// A reference outside an inner WITH reads the loaded table. +#[test] +fn loaded_table_before_inner_with() -> TestResult { + assert_query_on( + FILES, + "SELECT COUNT(*) FROM emp WHERE id IN (WITH emp AS (SELECT 2 AS id) SELECT id FROM emp)", + "COUNT(*)\n\ + 1", + ) +} + +// --------------------------------------------------------------------------- +// WITH RECURSIVE +// --------------------------------------------------------------------------- + +/// `WITH RECURSIVE` counter. +#[test] +fn recursive_counter() -> TestResult { + assert_query_on( + FILES, + "WITH RECURSIVE cnt(n) AS (SELECT 1 UNION ALL SELECT n + 1 FROM cnt WHERE n < 5) SELECT n FROM cnt", + "n\n\ + 1\n\ + 2\n\ + 3\n\ + 4\n\ + 5", + ) +} + +/// A self-referencing CTE is recursive without the keyword. +#[test] +fn self_reference_without_keyword_is_recursive() -> TestResult { + assert_query_on( + FILES, + "WITH cnt(n) AS (SELECT 1 UNION ALL SELECT n + 1 FROM cnt WHERE n < 5) SELECT n FROM cnt", + "n\n\ + 1\n\ + 2\n\ + 3\n\ + 4\n\ + 5", + ) +} + +/// Without a column list, the setup arm names the columns. +#[test] +fn recursive_names_from_setup() -> TestResult { + assert_query_on( + FILES, + "WITH RECURSIVE c AS (SELECT 1 AS n UNION ALL SELECT n + 1 FROM c WHERE n < 3) SELECT n FROM c", + "n\n\ + 1\n\ + 2\n\ + 3", + ) +} + +/// Breadth-first tree walk (FIFO queue). +#[test] +fn recursive_tree_walk() -> TestResult { + assert_query_on( + FILES, + "WITH RECURSIVE t(id, d) AS (SELECT id, 0 FROM tree WHERE parent IS NULL UNION ALL SELECT tree.id, t.d + 1 FROM tree JOIN t ON tree.parent = t.id) SELECT id, d FROM t", + "id,d\n\ + 1,0\n\ + 2,1\n\ + 3,1\n\ + 4,2\n\ + 5,2\n\ + 6,2\n\ + 7,3", + ) +} + +/// `ORDER BY 2 DESC` in the body makes the queue depth-first. +#[test] +fn recursive_depth_first() -> TestResult { + assert_query_on( + FILES, + "WITH RECURSIVE t(id, d) AS (SELECT id, 0 FROM tree WHERE parent IS NULL UNION ALL SELECT tree.id, t.d + 1 FROM tree JOIN t ON tree.parent = t.id ORDER BY 2 DESC) SELECT id, d FROM t", + "id,d\n\ + 1,0\n\ + 2,1\n\ + 4,2\n\ + 7,3\n\ + 5,2\n\ + 3,1\n\ + 6,2", + ) +} + +/// A body ORDER BY may name a setup column. +#[test] +fn recursive_order_by_name() -> TestResult { + assert_query_on( + FILES, + "WITH RECURSIVE t(id, d) AS (SELECT id, 0 AS depth FROM tree WHERE parent IS NULL UNION ALL SELECT tree.id, t.d + 1 FROM tree JOIN t ON tree.parent = t.id ORDER BY depth DESC, id DESC) SELECT id, d FROM t", + "id,d\n\ + 1,0\n\ + 3,1\n\ + 6,2\n\ + 2,1\n\ + 5,2\n\ + 4,2\n\ + 7,3", + ) +} + +/// Paths built by concatenation. +#[test] +fn recursive_paths() -> TestResult { + assert_query_on( + FILES, + "WITH RECURSIVE p(id, path) AS (SELECT id, CAST(id AS TEXT) FROM tree WHERE parent IS NULL UNION ALL SELECT tree.id, p.path || '/' || tree.id FROM tree JOIN p ON tree.parent = p.id) SELECT id, path FROM p ORDER BY path", + "id,path\n\ + 1,1\n\ + 2,1/2\n\ + 4,1/2/4\n\ + 7,1/2/4/7\n\ + 5,1/2/5\n\ + 3,1/3\n\ + 6,1/3/6", + ) +} + +/// UNION discards rows already produced, so a cycle terminates. +#[test] +fn recursive_union_dedupes() -> TestResult { + assert_query_on( + FILES, + "WITH RECURSIVE c(n) AS (SELECT 1 UNION SELECT n % 3 + 1 FROM c) SELECT n FROM c", + "n\n\ + 1\n\ + 2\n\ + 3", + ) +} + +/// Under UNION, duplicate setup rows enter the queue once. +#[test] +fn recursive_union_setup_dedupes() -> TestResult { + assert_query_on( + FILES, + "WITH RECURSIVE c(n) AS (SELECT 1 UNION ALL SELECT 1 UNION SELECT n + 1 FROM c WHERE n < 3) SELECT n FROM c", + "n\n\ + 1\n\ + 2\n\ + 3", + ) +} + +/// UNION ALL keeps every row. +#[test] +fn recursive_union_all_keeps_duplicates() -> TestResult { + assert_query_on( + FILES, + "WITH RECURSIVE c(n) AS (SELECT 1 UNION ALL SELECT 1 UNION ALL SELECT n + 1 FROM c WHERE n < 3) SELECT n FROM c", + "n\n\ + 1\n\ + 1\n\ + 2\n\ + 2\n\ + 3\n\ + 3", + ) +} + +/// LIMIT ends an otherwise infinite recursion. +#[test] +fn recursive_limit_stops_recursion() -> TestResult { + assert_query_on( + FILES, + "WITH RECURSIVE c(n) AS (SELECT 1 UNION ALL SELECT n + 1 FROM c LIMIT 4) SELECT n FROM c", + "n\n\ + 1\n\ + 2\n\ + 3\n\ + 4", + ) +} + +/// OFFSET skips output rows but they still recurse. +#[test] +fn recursive_limit_offset() -> TestResult { + assert_query_on( + FILES, + "WITH RECURSIVE c(n) AS (SELECT 1 UNION ALL SELECT n + 1 FROM c WHERE n < 6 LIMIT 3 OFFSET 2) SELECT n FROM c", + "n\n\ + 3\n\ + 4\n\ + 5", + ) +} + +/// Several UNION ALL recursive arms run for each row, in order. +#[test] +fn recursive_multiple_arms() -> TestResult { + assert_query_on( + FILES, + "WITH RECURSIVE c(n) AS (SELECT 1 UNION ALL SELECT n + 1 FROM c WHERE n < 3 UNION ALL SELECT n + 10 FROM c WHERE n < 3) SELECT n FROM c", + "n\n\ + 1\n\ + 2\n\ + 11\n\ + 3\n\ + 12", + ) +} + +/// DISTINCT in a recursive arm dedupes within one queue row's run only, so +/// two parents each yield the child. (Pins that the one-run-per-generation +/// evaluation is not used for such an arm.) +#[test] +fn recursive_arm_distinct_is_per_row() -> TestResult { + assert_query_on( + FILES, + "WITH RECURSIVE c(n) AS (SELECT 2 UNION ALL SELECT 3 UNION ALL SELECT DISTINCT 99 FROM c WHERE n < 10) SELECT n FROM c", + "n\n2\n3\n99\n99", + ) +} + +/// Each row's children follow it in queue order across a whole generation +/// (a binary tree of decimal paths). +#[test] +fn recursive_children_in_queue_order() -> TestResult { + assert_query_on( + FILES, + "WITH RECURSIVE c(n) AS (SELECT 1 UNION ALL SELECT n * 10 + k FROM c, (SELECT 1 AS k UNION ALL SELECT 2) d WHERE n < 100) SELECT n FROM c", + "n\n1\n11\n12\n111\n112\n121\n122", + ) +} + +/// Two-column recursion. +#[test] +fn recursive_fibonacci() -> TestResult { + assert_query_on( + FILES, + "WITH RECURSIVE f(a, b) AS (SELECT 0, 1 UNION ALL SELECT b, a + b FROM f WHERE b < 50) SELECT a FROM f", + "a\n\ + 0\n\ + 1\n\ + 1\n\ + 2\n\ + 3\n\ + 5\n\ + 8\n\ + 13\n\ + 21\n\ + 34", + ) +} + +/// A recursive CTE joined to itself. +#[test] +fn recursive_referenced_twice() -> TestResult { + assert_query_on( + FILES, + "WITH RECURSIVE c(n) AS (SELECT 1 UNION ALL SELECT n + 1 FROM c WHERE n < 3) SELECT c1.n, c2.n FROM c c1 JOIN c c2 ON c1.n = c2.n", + "n,n\n\ + 1,1\n\ + 2,2\n\ + 3,3", + ) +} + +/// A recursive CTE drives a DELETE. +#[test] +fn recursive_in_delete() -> TestResult { + assert_query_on( + FILES, + "WITH RECURSIVE s(id) AS (SELECT 2 UNION ALL SELECT tree.id FROM tree JOIN s ON tree.parent = s.id) DELETE FROM tree WHERE id IN (SELECT id FROM s); SELECT id FROM tree ORDER BY id", + "id\n\ + 1\n\ + 3\n\ + 6", + ) +} + +/// Aggregates over a recursive CTE are fine. +#[test] +fn recursive_with_aggregate_outside() -> TestResult { + assert_query_on( + FILES, + "WITH RECURSIVE c(n) AS (SELECT 1 UNION ALL SELECT n + 1 FROM c WHERE n < 100) SELECT COUNT(*), SUM(n) FROM c", + "COUNT(*),SUM(n)\n\ + 100,5050", + ) +} + +/// A recursive CTE inside a subquery. +#[test] +fn recursive_nested_in_subquery() -> TestResult { + assert_query_on( + FILES, + "SELECT id FROM tree WHERE id IN (WITH RECURSIVE s(x) AS (SELECT 3 UNION ALL SELECT tree.id FROM tree, s WHERE tree.parent = s.x) SELECT x FROM s) ORDER BY id", + "id\n\ + 3\n\ + 6", + ) +} + +// --------------------------------------------------------------------------- +// WITH RECURSIVE errors +// --------------------------------------------------------------------------- + +/// A recursive arm may reference the CTE once. +#[test] +fn recursive_two_direct_references() -> TestResult { + assert_query_on_fails( + FILES, + "WITH c(n) AS (SELECT 1 UNION ALL SELECT c.n + 1 FROM c, c AS d WHERE c.n < 3) SELECT * FROM c", + "multiple references to recursive table: c", + ) +} + +/// A recursive arm may not also reference the CTE in a subquery. +#[test] +fn recursive_reference_in_subquery() -> TestResult { + assert_query_on_fails( + FILES, + "WITH c(n) AS (SELECT 1 UNION ALL SELECT n + 1 FROM c WHERE n < 3 AND n IN (SELECT n FROM c)) SELECT * FROM c", + "multiple recursive references: c", + ) +} + +/// A self-reference only in a subquery is circular. +#[test] +fn self_reference_only_in_subquery() -> TestResult { + assert_query_on_fails( + FILES, + "WITH c AS (SELECT 1 UNION SELECT (SELECT COUNT(*) FROM c)) SELECT * FROM c", + "circular reference: c", + ) +} + +/// A self-reference in a derived table is circular. +#[test] +fn self_reference_in_derived_table() -> TestResult { + assert_query_on_fails( + FILES, + "WITH c(n) AS (SELECT 1 UNION ALL SELECT n + 1 FROM (SELECT * FROM c) WHERE n < 3) SELECT * FROM c", + "circular reference: c", + ) +} + +/// Recursion requires UNION or UNION ALL. +#[test] +fn self_reference_under_intersect() -> TestResult { + assert_query_on_fails( + FILES, + "WITH c(n) AS (SELECT 1 UNION ALL SELECT n + 1 FROM c INTERSECT SELECT 2) SELECT * FROM c", + "circular reference: c", + ) +} + +/// The setup (leftmost) arm may not reference the CTE. +#[test] +fn self_reference_in_setup() -> TestResult { + assert_query_on_fails( + FILES, + "WITH c(n) AS (SELECT n FROM c UNION ALL SELECT 1) SELECT * FROM c", + "circular reference: c", + ) +} + +/// A recursive arm may not aggregate. +#[test] +fn recursive_aggregate() -> TestResult { + assert_query_on_fails( + FILES, + "WITH c(n) AS (SELECT 1 UNION ALL SELECT COUNT(*) FROM c) SELECT * FROM c", + "recursive aggregate queries not supported", + ) +} + +/// A recursive arm may not GROUP BY. +#[test] +fn recursive_group_by() -> TestResult { + assert_query_on_fails( + FILES, + "WITH c(n) AS (SELECT 1 UNION ALL SELECT n + 1 FROM c GROUP BY n) SELECT * FROM c", + "recursive aggregate queries not supported", + ) +} + +/// A recursive arm may not use window functions. +#[test] +fn recursive_window() -> TestResult { + assert_query_on_fails( + FILES, + "WITH c(n) AS (SELECT 1 UNION ALL SELECT ROW_NUMBER() OVER () FROM c) SELECT * FROM c", + "cannot use window functions in recursive queries", + ) +} + +/// A recursive body's ORDER BY must name a result column. +#[test] +fn recursive_order_by_unknown() -> TestResult { + assert_query_on_fails( + FILES, + "WITH c(n) AS (SELECT 1 UNION ALL SELECT n + 1 FROM c WHERE n < 5 ORDER BY nosuch) SELECT * FROM c", + "1st ORDER BY term does not match any column in the result set", + ) +} + +/// A recursive body's ORDER BY position must be in range. +#[test] +fn recursive_order_by_out_of_range() -> TestResult { + assert_query_on_fails( + FILES, + "WITH c(n) AS (SELECT 1 UNION ALL SELECT n + 1 FROM c WHERE n < 5 ORDER BY 3) SELECT * FROM c", + "1st ORDER BY term out of range - should be between 1 and 1", + ) +} + +/// Every arm has the setup's column count. +#[test] +fn recursive_arm_column_count() -> TestResult { + assert_query_on_fails( + FILES, + "WITH c(n) AS (SELECT 1 UNION ALL SELECT n + 1, 2 FROM c WHERE n < 5) SELECT * FROM c", + "SELECTs to the left and right of UNION ALL do not have the same number of result columns", + ) +} + +/// A recursive body's LIMIT must be an integer. +#[test] +fn recursive_limit_must_be_integer() -> TestResult { + assert_query_on_fails( + FILES, + "WITH c(n) AS (SELECT 1 UNION ALL SELECT n + 1 FROM c WHERE n < 3 LIMIT 'x') SELECT * FROM c", + "datatype mismatch", + ) +} diff --git a/tests/defects/doc_review_2026_10.rs b/tests/defects/doc_review_2026_10.rs new file mode 100644 index 0000000..48e3537 --- /dev/null +++ b/tests/defects/doc_review_2026_10.rs @@ -0,0 +1,213 @@ +//! Red list: the code defects found by the 2026-10 documentation review. +//! +//! The review checked every claim in README.md, CLAUDE.md and doc/*.md +//! against the binary. Most findings were wrong documentation; the two below +//! were wrong behaviour, and every test in this module encodes the +//! **correct** behaviour for one of them. Each was `#[ignore]`d when written +//! and un-ignored as its defect was fixed; all of them now pass. A few tests +//! (marked in their doc comments) passed when written and pin behaviour the +//! fixes had to keep. +//! Run the red list with: +//! cargo test --test mod defects -- --ignored +//! +//! - A: `CREATE TABLE ... LOCATION 'x.tsv'` without `WITH (DELIMITER=...)` +//! wrote comma-separated text, while a `.tsv` / `.tab` file loads as +//! tab-separated, so the file did not load back as written. +//! - B: a declared column type other than INT/INTEGER, REAL/FLOAT/DOUBLE or +//! BOOL/BOOLEAN was recorded as TEXT. The declared name is now recorded as +//! written (shown by `.schema`) and its affinity follows SQLite's rules +//! (section 3.1 of SQLite's datatype documentation). + +use std::fs; +use std::io::Write; +use std::path::{Path, PathBuf}; +use std::process::{Command, Output, Stdio}; + +use crate::helpers::create_temp_dir; + +type TestResult = Result<(), Box>; + +fn sqawk_bin() -> PathBuf { + PathBuf::from(env!("CARGO_BIN_EXE_sqawk")) +} + +/// Run the sqawk binary with `args` in `cwd`, feeding `stdin`, with HOME and +/// XDG_STATE_HOME pointed at `cwd` so nothing touches the real home +/// directory. +fn run_in(cwd: &Path, args: &[&str], stdin: &[u8]) -> std::io::Result { + let mut child = Command::new(sqawk_bin()) + .args(args) + .current_dir(cwd) + .env("HOME", cwd) + .env("XDG_STATE_HOME", cwd) + .stdin(Stdio::piped()) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .spawn()?; + { + let mut si = child.stdin.take().unwrap(); + si.write_all(stdin)?; + } + child.wait_with_output() +} + +/// Run `args` in `cwd`, require success, and return stdout. +fn ok_in(cwd: &Path, args: &[&str], stdin: &[u8]) -> Result> { + let out = run_in(cwd, args, stdin)?; + assert!( + out.status.success(), + "sqawk {args:?} failed: {}", + String::from_utf8_lossy(&out.stderr) + ); + Ok(String::from_utf8_lossy(&out.stdout).replace("\r\n", "\n")) +} + +/// In a fresh directory holding `one.csv`, run `CREATE TABLE t (a INT, +/// b TEXT) LOCATION './'` plus two INSERTs under `--write`, then +/// return the written file's text and what a second run reading it back +/// (default delimiter rule) prints for `SELECT b, a`. +fn create_and_reload( + file: &str, + with: &str, +) -> Result<(String, String), Box> { + let dir = create_temp_dir()?; + fs::write(dir.path().join("one.csv"), "k\n1\n")?; + let sql = format!( + "CREATE TABLE t (a INT, b TEXT) LOCATION './{file}'{with}; \ + INSERT INTO t VALUES (1, 'p'), (2, 'q')" + ); + ok_in(dir.path(), &["--write", "-s", &sql, "one.csv"], b"")?; + let written = fs::read_to_string(dir.path().join(file))?.replace("\r\n", "\n"); + let stem = Path::new(file).file_stem().unwrap().to_str().unwrap(); + let reread = ok_in( + dir.path(), + &["-s", &format!("SELECT b, a FROM {stem}"), file], + b"", + )?; + Ok((written, reread)) +} + +/// Pipe `script` into `sqawk --interactive one.csv` in a fresh directory and +/// return stdout. +fn repl(script: &str) -> Result> { + let dir = create_temp_dir()?; + fs::write(dir.path().join("one.csv"), "k\n1\n")?; + ok_in(dir.path(), &["--interactive", "one.csv"], script.as_bytes()) +} + +// --------------------------------------------------------------------------- +// A: LOCATION's default delimiter follows the file extension +// --------------------------------------------------------------------------- + +/// A `.tsv` LOCATION is written tab-separated, so it loads back as written. +#[test] +fn location_tsv_is_written_tab_separated() -> TestResult { + let (written, reread) = create_and_reload("x.tsv", "")?; + assert_eq!(written, "a\tb\n1\tp\n2\tq\n"); + assert_eq!(reread, "b,a\np,1\nq,2\n"); + Ok(()) +} + +/// `.tab` follows the same rule as `.tsv`. +#[test] +fn location_tab_is_written_tab_separated() -> TestResult { + let (written, reread) = create_and_reload("y.tab", "")?; + assert_eq!(written, "a\tb\n1\tp\n2\tq\n"); + assert_eq!(reread, "b,a\np,1\nq,2\n"); + Ok(()) +} + +/// Control (passed when written): any other extension is comma-separated. +#[test] +fn location_csv_is_written_comma_separated() -> TestResult { + let (written, reread) = create_and_reload("c.csv", "")?; + assert_eq!(written, "a,b\n1,p\n2,q\n"); + assert_eq!(reread, "b,a\np,1\nq,2\n"); + Ok(()) +} + +/// Control (passed when written): an explicit `WITH (DELIMITER=...)` wins +/// over the extension. +#[test] +fn location_explicit_delimiter_wins_over_extension() -> TestResult { + let dir = create_temp_dir()?; + fs::write(dir.path().join("one.csv"), "k\n1\n")?; + ok_in( + dir.path(), + &[ + "--write", + "-s", + "CREATE TABLE t (a INT, b TEXT) LOCATION './z.tsv' WITH (DELIMITER=';'); \ + INSERT INTO t VALUES (1, 'p')", + "one.csv", + ], + b"", + )?; + let written = fs::read_to_string(dir.path().join("z.tsv"))?.replace("\r\n", "\n"); + assert_eq!(written, "a;b\n1;p\n"); + Ok(()) +} + +// --------------------------------------------------------------------------- +// B: declared column types are recorded as written +// --------------------------------------------------------------------------- + +/// `.schema` shows each declared type name as written. +#[test] +fn schema_shows_declared_type_names() -> TestResult { + let out = repl( + "CREATE TABLE q (a BIGINT, b DECIMAL(10,2), c VARCHAR(20), d INT, \ + e BOOLEAN, f BLOB, g DOUBLE PRECISION, h MONEY);\n.schema q\n", + )?; + assert_eq!( + out, + "CREATE TABLE q (\n a BIGINT,\n b DECIMAL(10,2),\n c VARCHAR(20),\n d INT,\n \ + e BOOLEAN,\n f BLOB,\n g DOUBLE PRECISION,\n h MONEY\n);\n" + ); + Ok(()) +} + +/// ALTER TABLE ... ADD COLUMN records its declared type as written too. +#[test] +fn alter_add_column_shows_declared_type_name() -> TestResult { + let out = repl("CREATE TABLE q (a INTEGER);\nALTER TABLE q ADD COLUMN b BIGINT;\n.schema q\n")?; + assert_eq!(out, "CREATE TABLE q (\n a INTEGER,\n b BIGINT\n);\n"); + Ok(()) +} + +/// CREATE TABLE ... AS names each column's type by its affinity: a column +/// of a declared table carries its declared type's SQLite affinity (INT +/// substring -> INTEGER; CHAR/CLOB/TEXT -> TEXT; BLOB -> BLOB; +/// REAL/FLOA/DOUB -> REAL; anything else -> NUMERIC), and a loaded column +/// or an expression keeps the type CTAS always gave it. +#[test] +fn ctas_column_types_follow_declared_affinity() -> TestResult { + let out = repl( + "CREATE TABLE q (a BIGINT, b DECIMAL(10,2), c VARCHAR(20), e BOOLEAN, \ + f BLOB, g DOUBLE PRECISION, h CHARINT);\n\ + CREATE TABLE r AS SELECT a, b, c, e, f, g, h, k, LENGTH(k) AS n FROM q, one;\n\ + .schema r\n", + )?; + assert_eq!( + out, + "CREATE TABLE r (\n a INTEGER,\n b NUMERIC,\n c TEXT,\n e NUMERIC,\n f BLOB,\n \ + g REAL,\n h INTEGER,\n k TEXT,\n n INTEGER\n);\n" + ); + Ok(()) +} + +/// Control (passed when written): a declared type is not enforced -- every +/// value keeps its own type. +#[test] +fn declared_types_are_not_enforced() -> TestResult { + let out = repl( + "CREATE TABLE q (a BIGINT, b DECIMAL(10,2), c VARCHAR(2));\n\ + INSERT INTO q VALUES ('abc', 'x', 12345);\n\ + SELECT a, typeof(a), b, typeof(b), c, typeof(c) FROM q;\n", + )?; + assert_eq!( + out, + "a,typeof(a),b,typeof(b),c,typeof(c)\nabc,text,x,text,12345,integer\n" + ); + Ok(()) +} diff --git a/tests/defects/mod.rs b/tests/defects/mod.rs index ebb20f0..95fc2f1 100644 --- a/tests/defects/mod.rs +++ b/tests/defects/mod.rs @@ -5,11 +5,18 @@ //! it wrong, and un-ignored as it was fixed. //! //! Every 2026-08 test here now passes, and so does every test of the -//! `audit_2026_10` submodule (defects from the 2026-10 audit) and of the +//! `audit_2026_10` submodule (defects from the 2026-10 audit), of the //! `code_review_2026_10` submodule (defects from the 2026-10 code review of -//! the `updates` branch): the red list is EMPTY. All serve as regression -//! cover rather than as a to-do list. Add a new `#[ignore]`d test when a -//! defect is found, and remove the attribute when it is fixed. +//! the `updates` branch), of the `v1_review_2026_10` submodule (findings of +//! the 2026-10 v1-readiness review) and of the `doc_review_2026_10` +//! submodule (code defects found by the 2026-10 documentation review). All +//! serve as regression cover. The `schema_2026_10` submodule holds the +//! `CREATE TABLE ... LOCATION` over an existing file defect, and the +//! `cte_2026_10` submodule the common table expression (`WITH`) rules. The +//! `tsq_2026_10` submodule holds the defects found by `tsq diff`, the +//! differential tester against SQLite (still red). Add a +//! new `#[ignore]`d test when a defect is found, and remove the attribute +//! when it is fixed. //! //! Run the red list with: //! cargo test --test mod defects -- --ignored @@ -22,6 +29,11 @@ mod audit_2026_10; mod code_review_2026_10; +mod cte_2026_10; +mod doc_review_2026_10; +mod schema_2026_10; +mod tsq_2026_10; +mod v1_review_2026_10; use crate::helpers::{ assert_after_write, assert_query, departments_csv, employees_csv, nullable_csv, orders_csv, @@ -40,7 +52,7 @@ fn where_is_applied_to_group_by() -> Result<(), Box> { assert_query( &[employees_csv()], "SELECT department, COUNT(*) FROM employees WHERE salary > 60000 GROUP BY department", - "department,COUNT\n\ + "department,COUNT(*)\n\ Engineering,3\n\ Sales,2", ) @@ -55,7 +67,7 @@ fn group_by_uses_all_key_columns() -> Result<(), Box> { assert_query( &[employees_csv()], "SELECT department, role, COUNT(*) FROM employees GROUP BY department, role", - "department,role,COUNT\n\ + "department,role,COUNT(*)\n\ Engineering,Developer,2\n\ Engineering,Manager,1\n\ HR,Intern,1\n\ @@ -72,7 +84,7 @@ fn order_by_applies_to_group_by() -> Result<(), Box> { assert_query( &[employees_csv()], "SELECT department, COUNT(*) FROM employees GROUP BY department ORDER BY department DESC", - "department,COUNT\n\ + "department,COUNT(*)\n\ Sales,2\n\ Marketing,2\n\ HR,1\n\ @@ -86,7 +98,7 @@ fn limit_applies_to_group_by() -> Result<(), Box> { assert_query( &[employees_csv()], "SELECT department, COUNT(*) FROM employees GROUP BY department LIMIT 1", - "department,COUNT\n\ + "department,COUNT(*)\n\ Engineering,3", ) } @@ -100,7 +112,7 @@ fn having_binds_to_the_named_aggregate() -> Result<(), Box 60", - "department,SUM,SUM\n\ + "department,SUM(salary),SUM(age)\n\ Engineering,210000,97\n\ Sales,170000,85", ) @@ -112,7 +124,7 @@ fn count_distinct_deduplicates() -> Result<(), Box> { assert_query( &[employees_csv()], "SELECT COUNT(DISTINCT department) FROM employees", - "COUNT\n4", + "COUNT(DISTINCT department)\n4", ) } @@ -125,7 +137,7 @@ fn group_by_respects_projection_order() -> Result<(), Box assert_query( &[employees_csv()], "SELECT COUNT(*), department FROM employees GROUP BY department", - "COUNT,department\n\ + "COUNT(*),department\n\ 3,Engineering\n\ 1,HR\n\ 2,Marketing\n\ @@ -140,7 +152,7 @@ fn group_by_emits_only_projected_columns() -> Result<(), Box Result<(), Box> { assert_query( &[users_csv(), orders_csv()], "SELECT orders.id, users.name FROM users JOIN orders ON users.id = orders.user_id", - "orders.id,users.name\n\ + "id,name\n\ 101,John\n\ 103,John\n\ 105,John\n\ @@ -165,15 +177,14 @@ fn join_respects_projection_order() -> Result<(), Box> { ) } -/// Defect 4: ORDER BY is deliberately skipped on explicit JOINs -/// (compiler.rs:729-737 -- "isn't fully implemented. For now [...] skip"). +/// Defect 4: ORDER BY was skipped on explicit JOINs. #[test] fn order_by_applies_to_join() -> Result<(), Box> { assert_query( &[users_csv(), orders_csv()], "SELECT users.name, orders.id FROM users JOIN orders ON users.id = orders.user_id \ ORDER BY orders.id DESC", - "users.name,orders.id\n\ + "name,id\n\ John,105\n\ Jane,104\n\ John,103\n\ @@ -195,7 +206,7 @@ fn arithmetic_projection_survives_limit() -> Result<(), Box Result<(), Box> { assert_query( &[employees_csv()], "SELECT name, CASE WHEN age > 30 THEN 'old' ELSE 'young' END FROM employees LIMIT 3", - "name,expr\n\ + "name,CASE WHEN age > 30 THEN 'old' ELSE 'young' END\n\ Alice,young\n\ Bob,young\n\ Charlie,old", @@ -223,7 +234,7 @@ fn string_concat_operator_works() -> Result<(), Box> { assert_query( &[employees_csv()], "SELECT name || '!' FROM employees LIMIT 2", - "expr\n\ + "name || '!'\n\ Alice!\n\ Bob!", ) @@ -235,7 +246,7 @@ fn comparison_works_in_projection() -> Result<(), Box> { assert_query( &[employees_csv()], "SELECT name, salary > 60000 FROM employees LIMIT 3", - "name,expr\n\ + "name,salary > 60000\n\ Alice,1\n\ Bob,0\n\ Charlie,1", @@ -249,7 +260,7 @@ fn aggregate_accepts_expression_argument() -> Result<(), Box Result<(), Box Result<(), Box Result<(), Box Result<(), Box Result<(), Box> { assert_query( &[users_csv(), orders_csv()], "SELECT COUNT(*) FROM users JOIN orders ON users.id = orders.user_id", - "COUNT\n5", + "COUNT(*)\n5", ) } diff --git a/tests/defects/schema_2026_10.rs b/tests/defects/schema_2026_10.rs new file mode 100644 index 0000000..3b944d6 --- /dev/null +++ b/tests/defects/schema_2026_10.rs @@ -0,0 +1,441 @@ +//! Red list: `CREATE TABLE ... LOCATION` over a file that already exists. +//! +//! `CREATE TABLE t (id INT, name TEXT) LOCATION './existing.csv'` ignored the +//! file's contents (the table started empty) and `--write` then replaced the +//! file with only the rows the session inserted -- or with a bare header when +//! no statement modified the table. Every test in this module encodes the +//! **correct** behaviour, which follows Hive external-table semantics: the +//! declared columns are the schema for the existing file. +//! +//! - An existing LOCATION file is loaded through the normal loader under the +//! declared column names. Its first line is a header unless +//! `WITH (HEADER=false)`: it is skipped as data, must have one field per +//! declared column, and is written back verbatim. +//! - The delimiter is `WITH (DELIMITER=...)`, else tab for `.tsv` / `.tab` +//! and comma for everything else. +//! - Every untouched row is written back byte-for-byte. +//! - A LOCATION that does not exist is a new, empty table, created on +//! `--write`. +//! - A file already loaded (as an operand or another table) is an error. +//! - ROLLBACK removes the table and never touches the file. +//! +//! Each test was `#[ignore]`d when written and un-ignored when the defect +//! was fixed; all of them now pass. A few tests (marked in their doc +//! comments) passed when written and pin behaviour the fix had to keep, and +//! the last two were added with the fix. Run the red list with: +//! cargo test --test mod defects -- --ignored + +use std::fs; +use std::path::{Path, PathBuf}; +use std::process::{Command, Output}; + +use crate::helpers::create_temp_dir; + +type TestResult = Result<(), Box>; + +/// Run the sqawk binary with `args` in `cwd` (HOME pointed there too). +fn run_in(cwd: &Path, args: &[&str]) -> std::io::Result { + Command::new(PathBuf::from(env!("CARGO_BIN_EXE_sqawk"))) + .args(args) + .current_dir(cwd) + .env("HOME", cwd) + .env("XDG_STATE_HOME", cwd) + .output() +} + +/// Run `sql` in `cwd` with the file operand `one.csv` (created here), +/// plus `--write` when `write`; require success and return stdout. +fn ok(cwd: &Path, sql: &str, write: bool) -> Result> { + fs::write(cwd.join("one.csv"), "k\n1\n")?; + let mut args = vec!["-s", sql, "one.csv"]; + if write { + args.insert(0, "--write"); + } + let out = run_in(cwd, &args)?; + assert!( + out.status.success(), + "sqawk {args:?} failed: {}", + String::from_utf8_lossy(&out.stderr) + ); + Ok(String::from_utf8_lossy(&out.stdout).replace("\r\n", "\n")) +} + +/// Run `args` in `cwd`, require failure, and return stderr. +fn fails(cwd: &Path, args: &[&str]) -> Result> { + let out = run_in(cwd, args)?; + assert!( + !out.status.success(), + "sqawk {args:?} succeeded: {}", + String::from_utf8_lossy(&out.stdout) + ); + Ok(String::from_utf8_lossy(&out.stderr).into_owned()) +} + +/// The existing file's rows are the table's rows, under the declared names. +#[test] +fn existing_file_rows_are_visible() -> TestResult { + let dir = create_temp_dir()?; + fs::write(dir.path().join("e.csv"), "num,label\n1,alice\n2,bob\n")?; + let out = ok( + dir.path(), + "CREATE TABLE t (id INT, name TEXT) LOCATION './e.csv'; SELECT id, name FROM t", + false, + )?; + assert_eq!(out, "id,name\n1,alice\n2,bob\n"); + Ok(()) +} + +/// The declared types are the columns' types: `id INT` compares numerically. +#[test] +fn existing_file_uses_declared_types() -> TestResult { + let dir = create_temp_dir()?; + fs::write(dir.path().join("e.csv"), "num,label\n10,a\n9,b\n")?; + let out = ok( + dir.path(), + "CREATE TABLE t (id INT, name TEXT) LOCATION './e.csv'; \ + SELECT name FROM t ORDER BY id", + false, + )?; + assert_eq!(out, "name\nb\na\n"); + Ok(()) +} + +/// With no DML, `--write` leaves the file byte-identical (BOM, CRLF, quoting +/// and the header's own text included). +#[test] +fn no_dml_write_leaves_file_identical() -> TestResult { + let dir = create_temp_dir()?; + let data = b"\xef\xbb\xbfnum,\"label\"\r\n1,\"a b\"\r\n2, bob\r\n"; + fs::write(dir.path().join("e.csv"), data)?; + ok( + dir.path(), + "CREATE TABLE t (id INT, name TEXT) LOCATION './e.csv'", + true, + )?; + assert_eq!(fs::read(dir.path().join("e.csv"))?, data.to_vec()); + Ok(()) +} + +/// INSERT appends; the original header and rows stay byte-identical. +#[test] +fn insert_write_appends_and_keeps_rows() -> TestResult { + let dir = create_temp_dir()?; + fs::write( + dir.path().join("e.csv"), + "num,label\n1,\"alice\"\n2, bob\n", + )?; + ok( + dir.path(), + "CREATE TABLE t (id INT, name TEXT) LOCATION './e.csv'; \ + INSERT INTO t VALUES (3, 'carol')", + true, + )?; + assert_eq!( + fs::read_to_string(dir.path().join("e.csv"))?, + "num,label\n1,\"alice\"\n2, bob\n3,carol\n" + ); + Ok(()) +} + +/// UPDATE of one row rewrites that row only. +#[test] +fn update_one_row_keeps_others() -> TestResult { + let dir = create_temp_dir()?; + fs::write( + dir.path().join("e.csv"), + "num,label\n1,\"alice\"\n2, bob\n3,x\n", + )?; + ok( + dir.path(), + "CREATE TABLE t (id INT, name TEXT) LOCATION './e.csv'; \ + UPDATE t SET name = 'y' WHERE id = 3", + true, + )?; + assert_eq!( + fs::read_to_string(dir.path().join("e.csv"))?, + "num,label\n1,\"alice\"\n2, bob\n3,y\n" + ); + Ok(()) +} + +/// A header whose field count differs from the declared column count is an +/// error naming the file and both counts; the file is left alone. +#[test] +fn header_field_count_mismatch_is_an_error() -> TestResult { + let dir = create_temp_dir()?; + let data = "a,b,c\n1,2,3\n"; + fs::write(dir.path().join("e.csv"), data)?; + fs::write(dir.path().join("one.csv"), "k\n1\n")?; + let err = fails( + dir.path(), + &[ + "--write", + "-s", + "CREATE TABLE t (id INT, name TEXT) LOCATION './e.csv'; \ + INSERT INTO t VALUES (9, 'z')", + "one.csv", + ], + )?; + assert!(err.contains("e.csv"), "{err}"); + assert!( + err.contains("3 fields") && err.contains("2 columns"), + "{err}" + ); + assert_eq!(fs::read_to_string(dir.path().join("e.csv"))?, data); + Ok(()) +} + +/// `WITH (HEADER=false)`: the first line is data, the columns are the +/// declared ones, and writeback emits no header. +#[test] +fn header_false_first_line_is_data() -> TestResult { + let dir = create_temp_dir()?; + fs::write(dir.path().join("d.txt"), "1,alice\n2,bob\n")?; + let out = ok( + dir.path(), + "CREATE TABLE t (id INT, name TEXT) LOCATION './d.txt' WITH (HEADER=false); \ + SELECT * FROM t", + false, + )?; + assert_eq!(out, "id,name\n1,alice\n2,bob\n"); + ok( + dir.path(), + "CREATE TABLE t (id INT, name TEXT) LOCATION './d.txt' WITH (HEADER=FALSE); \ + INSERT INTO t VALUES (3, 'carol')", + true, + )?; + assert_eq!( + fs::read_to_string(dir.path().join("d.txt"))?, + "1,alice\n2,bob\n3,carol\n" + ); + Ok(()) +} + +/// `WITH (HEADER=true)` is the default spelled out. +#[test] +fn header_true_is_the_default() -> TestResult { + let dir = create_temp_dir()?; + fs::write(dir.path().join("e.csv"), "num,label\n1,alice\n")?; + let out = ok( + dir.path(), + "CREATE TABLE t (id INT, name TEXT) LOCATION './e.csv' WITH (header='True'); \ + SELECT * FROM t", + false, + )?; + assert_eq!(out, "id,name\n1,alice\n"); + Ok(()) +} + +/// A `.tsv` LOCATION is read tab-separated. +#[test] +fn tsv_extension_reads_tab_separated() -> TestResult { + let dir = create_temp_dir()?; + fs::write(dir.path().join("x.tsv"), "num\tlabel\n1\talice, a\n")?; + let out = ok( + dir.path(), + "CREATE TABLE t (id INT, name TEXT) LOCATION './x.tsv'; \ + INSERT INTO t VALUES (2, 'bob'); SELECT name FROM t", + true, + )?; + assert_eq!(out, "name\n\"alice, a\"\nbob\n"); + assert_eq!( + fs::read_to_string(dir.path().join("x.tsv"))?, + "num\tlabel\n1\talice, a\n2\tbob\n" + ); + Ok(()) +} + +/// `WITH (DELIMITER=...)` overrides the extension rule. +#[test] +fn with_delimiter_overrides_extension() -> TestResult { + let dir = create_temp_dir()?; + fs::write(dir.path().join("x.csv"), "num;label\n1;alice\n")?; + let out = ok( + dir.path(), + "CREATE TABLE t (id INT, name TEXT) LOCATION './x.csv' WITH (DELIMITER=';'); \ + SELECT name, id FROM t", + false, + )?; + assert_eq!(out, "name,id\nalice,1\n"); + Ok(()) +} + +/// `WITH (COMMENTS=true)`: `#` lines are skipped and written back in place. +#[test] +fn with_comments_skips_and_keeps_hash_lines() -> TestResult { + let dir = create_temp_dir()?; + fs::write( + dir.path().join("c.csv"), + "# top\nnum,label\n1,alice\n# mid\n2,bob\n", + )?; + let out = ok( + dir.path(), + "CREATE TABLE t (id INT, name TEXT) LOCATION './c.csv' WITH (COMMENTS=true); \ + DELETE FROM t WHERE id = 1; SELECT * FROM t", + true, + )?; + assert_eq!(out, "id,name\n2,bob\n"); + assert_eq!( + fs::read_to_string(dir.path().join("c.csv"))?, + "# top\nnum,label\n# mid\n2,bob\n" + ); + Ok(()) +} + +/// A record with more fields than declared columns is the usual load error, +/// with file and line. +#[test] +fn too_many_fields_is_a_load_error() -> TestResult { + let dir = create_temp_dir()?; + fs::write(dir.path().join("e.csv"), "num,label\n1,a\n2,b,c\n")?; + fs::write(dir.path().join("one.csv"), "k\n1\n")?; + let err = fails( + dir.path(), + &[ + "-s", + "CREATE TABLE t (id INT, name TEXT) LOCATION './e.csv'", + "one.csv", + ], + )?; + assert!(err.contains("e.csv") && err.contains('3'), "{err}"); + Ok(()) +} + +/// Passed when written: a LOCATION that does not exist is a new table, +/// created on `--write` with a header of the declared names. +#[test] +fn missing_file_is_created() -> TestResult { + let dir = create_temp_dir()?; + ok( + dir.path(), + "CREATE TABLE t (id INT, name TEXT) LOCATION './new.csv'; \ + INSERT INTO t VALUES (1, 'a')", + true, + )?; + assert_eq!( + fs::read_to_string(dir.path().join("new.csv"))?, + "id,name\n1,a\n" + ); + Ok(()) +} + +/// A new file under `WITH (HEADER=false)` is written without a header. +#[test] +fn missing_file_header_false_has_no_header() -> TestResult { + let dir = create_temp_dir()?; + ok( + dir.path(), + "CREATE TABLE t (id INT, name TEXT) LOCATION './new.csv' WITH (HEADER=false); \ + INSERT INTO t VALUES (1, 'a')", + true, + )?; + assert_eq!(fs::read_to_string(dir.path().join("new.csv"))?, "1,a\n"); + Ok(()) +} + +/// The LOCATION file is already loaded as an operand: an error, and the +/// file is left alone. +#[test] +fn location_already_loaded_as_operand() -> TestResult { + let dir = create_temp_dir()?; + let data = "id,name\n1,alice\n"; + fs::write(dir.path().join("e.csv"), data)?; + let err = fails( + dir.path(), + &[ + "--write", + "-s", + "CREATE TABLE t (id INT, name TEXT) LOCATION './e.csv'; \ + INSERT INTO t VALUES (2, 'bob')", + "e.csv", + ], + )?; + assert!(err.contains("already loaded as table e"), "{err}"); + assert_eq!(fs::read_to_string(dir.path().join("e.csv"))?, data); + Ok(()) +} + +/// Two LOCATION tables over one file: the second is an error. +#[test] +fn location_already_loaded_as_another_table() -> TestResult { + let dir = create_temp_dir()?; + fs::write(dir.path().join("e.csv"), "id,name\n1,alice\n")?; + fs::write(dir.path().join("one.csv"), "k\n1\n")?; + let err = fails( + dir.path(), + &[ + "-s", + "CREATE TABLE t (id INT, name TEXT) LOCATION './e.csv'; \ + CREATE TABLE u (id INT, name TEXT) LOCATION 'e.csv'", + "one.csv", + ], + )?; + assert!(err.contains("already loaded as table t"), "{err}"); + Ok(()) +} + +/// Passed when written: ROLLBACK removes the table and does not touch the +/// file. +#[test] +fn rollback_leaves_file_untouched() -> TestResult { + let dir = create_temp_dir()?; + let data = "num,label\n1,alice\n"; + fs::write(dir.path().join("e.csv"), data)?; + ok( + dir.path(), + "BEGIN; CREATE TABLE t (id INT, name TEXT) LOCATION './e.csv'; \ + INSERT INTO t VALUES (2, 'bob'); ROLLBACK", + true, + )?; + assert_eq!(fs::read_to_string(dir.path().join("e.csv"))?, data); + Ok(()) +} + +/// A LOCATION that is a directory is a clean error. +#[test] +fn location_directory_is_an_error() -> TestResult { + let dir = create_temp_dir()?; + fs::create_dir(dir.path().join("sub"))?; + fs::write(dir.path().join("one.csv"), "k\n1\n")?; + let err = fails( + dir.path(), + &[ + "-s", + "CREATE TABLE t (id INT) LOCATION './sub'; SELECT * FROM t", + "one.csv", + ], + )?; + assert!(err.starts_with("sqawk: ") && err.contains("sub"), "{err}"); + Ok(()) +} + +/// `HEADER` and `COMMENTS` take true or false (any case, quoted or not); +/// anything else is an error that names the option. +#[test] +fn boolean_option_value_is_checked() -> TestResult { + let dir = create_temp_dir()?; + fs::write(dir.path().join("one.csv"), "k\n1\n")?; + for with in ["HEADER=maybe", "COMMENTS='yes'", "header=1"] { + let sql = format!("CREATE TABLE t (id INT) LOCATION './n.csv' WITH ({with})"); + let err = fails(dir.path(), &["-s", &sql, "one.csv"])?; + assert!(err.contains("must be true or false"), "{with}: {err}"); + } + Ok(()) +} + +/// TRUNCATE leaves the file's own header line, not the declared names. +#[test] +fn truncate_keeps_the_header_line() -> TestResult { + let dir = create_temp_dir()?; + fs::write(dir.path().join("e.csv"), "NUM,\"Label\"\n1,a\n")?; + ok( + dir.path(), + "CREATE TABLE t (id INT, name TEXT) LOCATION './e.csv'; TRUNCATE TABLE t", + true, + )?; + assert_eq!( + fs::read_to_string(dir.path().join("e.csv"))?, + "NUM,\"Label\"\n" + ); + Ok(()) +} diff --git a/tests/defects/tsq_2026_10.rs b/tests/defects/tsq_2026_10.rs new file mode 100644 index 0000000..6d262ae --- /dev/null +++ b/tests/defects/tsq_2026_10.rs @@ -0,0 +1,348 @@ +//! Red list: defects found by `tsq diff` (2026-10), the differential tester +//! that runs random SQL through sqawk and through SQLite 3.45, and by its +//! fuzzers `tsq wb` and `tsq cli`. +//! +//! Every test encodes **correct** behaviour: the answer SQLite 3.45 gives +//! (python3 `sqlite3` 3.45.1 and rusqlite's bundled 3.45.0 agree on each), +//! which is the behaviour sqawk documents (doc/sql_reference.md: "Semantics +//! follow SQLite 3.45"; none of these is a listed difference), or for a +//! fuzzer finding the documented command-line behaviour. Each is +//! `#[ignore]`d until fixed; remove the attribute with the fix. Run with: +//! cargo test --test mod defects -- --ignored +//! +//! Each test names the tsq seed and test number that found it; `tsq diff +//! --seed S --only N --shrink` replays and minimizes it. + +use std::time::Duration; + +use crate::helpers::{assert_query_on, create_temp_dir}; + +type TestResult = Result<(), Box>; + +const ONE: &[(&str, &str)] = &[("t.csv", "x\n1\n")]; + +/// A real prints as SQLite's `CAST(x AS TEXT)` (doc "Real output"), which +/// rounds a decimal expansion of about 18 digits half up to 15 digits. The +/// double nearest 443.85 / 5867 is 0.07565195159365944999..., whose 18-digit +/// expansion ends ...4945|00 and rounds up to ...595; sqawk rounded the exact +/// binary value and printed ...594. Found by tsq diff --seed 1 --only 19555 +/// (also PERCENT_RANK values at --scale large, seed 1 test 26). +/// +/// The expansion is SQLite's double-double one (3.50, and 3.45 built +/// without long double as tsq's oracle is): 100.0 / 476.42 is ...536 there +/// (...535 with x87 long double). +#[test] +fn real_text_rounds_like_sqlite() -> TestResult { + assert_query_on( + ONE, + "SELECT 443.85 / 5867 AS r, CAST(443.85 / 5867 AS TEXT) AS t, \ + 443.85 / 5867 || '' AS c, 100.0 / 476.42 AS d FROM t", + "r,t,c,d\n0.0756519515936595,0.0756519515936595,0.0756519515936595,0.209898828764536", + ) +} + +/// QUOTE of a real whose 15-digit form does not read back as the same value +/// falls back to SQLite's `%!.20e`, which prints the digits SQLite's decoder +/// produces (`6.44491850400551701e-04`). sqawk dropped the trailing digits +/// (`6.444918504005517e-04`). Found by tsq diff --seed 1 --only 17090. The +/// digits are the double-double ones (SQLite 3.50; 3.45 without long double): +/// `...392e+01` and `...3493e+09` where x87 long double gives `...39e+01` and +/// `...349e+09`. "Reads back" is SQLite's own text-to-real reading, which +/// for 1e-300 gives the double that `1.0e-300` gives, so it quotes short. +#[test] +fn quote_real_fallback_digits() -> TestResult { + assert_query_on( + ONE, + "SELECT QUOTE(1 / 1551.61) AS a, QUOTE(17 / 1.5) AS b, QUOTE(1e10 / 3) AS c, \ + QUOTE(1e-300) AS d FROM t", + "a,b,c,d\n6.44491850400551701e-04,1.133333333333333392e+01,3.333333333333333493e+09,1.0e-300", + ) +} + +/// `%` with a real operand takes the remainder of the operands truncated to +/// integers (doc "Arithmetic"), and integers beyond 2^53 stay exact: the +/// integer operand must not pass through a double. sqawk rounded +/// 9007199254740993 to ...992 first (`% 2.0` gave 0.0, not 1.0); an +/// out-of-range real clamps to the i64 limit as in SQLite. Found by tsq diff +/// --seed 2 --only 175. +#[test] +fn remainder_with_real_keeps_integers_exact() -> TestResult { + assert_query_on( + ONE, + "SELECT 9007199254740993 % 2.0 AS a, 9007199254740993 % 10.0 AS b, \ + -9223372036854775793 % -2.67e20 AS c FROM t", + "a,b,c\n1.0,3.0,-9.22337203685478e+18", + ) +} + +/// ROUND judges halves "on the value actually stored" (doc "Math +/// Functions": `ROUND(2.675, 2)` is 2.67). 1058.05 is stored as +/// 1058.04999999999995..., so ROUND(1058.05, 1) is 1058.0 in SQLite; sqawk +/// gave 1058.1. Found by tsq diff --seed 4 --only 2989. +#[test] +fn round_judges_the_stored_value() -> TestResult { + assert_query_on( + ONE, + "SELECT ROUND(1058.05, 1) AS a, ROUND(-1058.05, 1) AS b, ROUND(0.285, 2) AS c FROM t", + "a,b,c\n1058.0,-1058.0,0.28", + ) +} + +// --------------------------------------------------------------------------- +// Performance cliffs. The answers are right; the time is the defect: each +// query finishes in milliseconds in SQLite and takes quadratic time in +// sqawk, so `tsq diff --scale large` reports it as a crash (its 10 s limit). +// The data is sized so the quadratic code needs over a minute (debug build) +// while linear code needs well under a second; the 20 s limit leaves room +// for slow runners without letting the quadratic code pass. +// --------------------------------------------------------------------------- + +/// Run `sql` over `files` (written to a temp dir), killed after 20 s, and +/// require the exact stdout. +fn assert_fast(files: &[(&str, String)], sql: &str, expected: &str) -> TestResult { + let dir = create_temp_dir()?; + let mut cmd = assert_cmd::cargo::cargo_bin_cmd!("sqawk"); + cmd.arg("-s").arg(sql); + for (name, content) in files { + let path = dir.path().join(name); + std::fs::write(&path, content)?; + cmd.arg(path); + } + let out = cmd.timeout(Duration::from_secs(20)).output()?; + assert!( + out.status.success(), + "sqawk did not finish within 20 s ({})\n sql: {sql}", + out.status + ); + assert_eq!(String::from_utf8_lossy(&out.stdout), expected); + Ok(()) +} + +fn csv(header: &str, rows: impl Iterator) -> String { + let mut s = format!("{header}\n"); + for r in rows { + s.push_str(&r); + s.push('\n'); + } + s +} + +/// An EXISTS subquery whose only tie to the outer row is a predicate on the +/// outer columns (`t1.b < t1.c`) must not rescan the inner table for every +/// outer row: SQLite evaluates such a term once per outer row and stops the +/// subquery at its first row; sqawk was O(n^2) (about 95 s for these 12000 +/// rows in a debug build, 14 s in a release build; 0.04 s once fixed). Found by tsq diff --scale large --seed 1 +/// --only 35. +#[test] +fn exists_with_outer_only_predicate_is_not_quadratic() -> TestResult { + let r = csv( + "id,a,b,c", + (1..=12000).map(|i| format!("{i},{},w{},w{}", i % 7, i % 13, i % 11)), + ); + assert_fast( + &[("r.csv", r)], + "SELECT COUNT(*) FROM r t1 WHERE EXISTS \ + (SELECT 1 FROM r t3 WHERE t3.a < 19 AND t1.b < t1.c)", + "COUNT(*)\n5962\n", + ) +} + +/// A window frame that ends at UNBOUNDED FOLLOWING while its start moves +/// (`ROWS BETWEEN 2 PRECEDING AND UNBOUNDED FOLLOWING`, `RANGE BETWEEN +/// CURRENT ROW AND UNBOUNDED FOLLOWING`) was recomputed from scratch for +/// every row in sqawk: O(n^2), about 77 s for these 60000 rows in a debug +/// build (0.2 s once fixed) where SQLite takes 0.01 s (it removes rows +/// leaving the frame instead). Found by tsq diff --scale large --seed 7 +/// (tests 277 and 392). +#[test] +fn window_frame_to_unbounded_following_is_not_quadratic() -> TestResult { + let w = csv("id,x", (1..=60000).map(|i| format!("{i},{}", i % 97))); + assert_fast( + &[("w.csv", w)], + "SELECT SUM(c) AS s, MAX(m) AS m FROM (SELECT \ + COUNT(x) OVER (ORDER BY id ROWS BETWEEN 2 PRECEDING AND UNBOUNDED FOLLOWING) AS c, \ + MAX(x) OVER (ORDER BY id RANGE BETWEEN CURRENT ROW AND UNBOUNDED FOLLOWING) AS m \ + FROM w)", + "s,m\n1800149997,96\n", + ) +} + +/// A WHERE term on the first table of a join that is an `IN (SELECT ...)` +/// was applied only after the whole join was built, while a plain +/// comparison on the same column is applied at that table's loop. Here the join is +/// 40 million rows unfiltered (93 s in a debug build) and 40000 filtered +/// (0.4 s); SQLite takes 0.2 s. +/// Found by tsq diff --scale large --seed 7 --only 233 (and --seed 3 +/// --only 12902 at small scale). +#[test] +fn in_subquery_filter_applies_before_the_joins() -> TestResult { + let p = csv("pid,q", (1..=1000).map(|i| format!("{i},{i}"))); + let it = csv( + "iid,pid", + (1..=200000).map(|i| format!("{i},{}", i % 1000 + 1)), + ); + assert_fast( + &[("p.csv", p), ("it.csv", it)], + "SELECT COUNT(*) FROM p t1 JOIN it t2 ON t1.pid = t2.pid \ + JOIN it t3 ON t1.pid = t3.pid WHERE t1.pid IN (SELECT pid FROM p WHERE q = 7)", + "COUNT(*)\n40000\n", + ) +} + +/// A CTE is computed when a query first reads it, and a WHERE term that +/// reads no table is tested before the FROM tables are opened: here `d` is +/// read only by an arm whose WHERE is constant false ('D' sorts before +/// 'abc'), so SQLite never computes `d` -- nor `c`, read only by `d`. sqawk +/// computed every referenced CTE before the statement ran: `c` is 26667 rows +/// and its self-join in `d` 118 million (18 s in a release build, minutes +/// in a debug build); SQLite takes 0.001 s. Found by tsq diff --seed 4 --only +/// 3486 (91 s there). +#[test] +fn cte_read_only_under_a_false_where_is_never_computed() -> TestResult { + let t = csv("id,k", (1..=400).map(|i| format!("{i},{}", i % 6))); + assert_fast( + &[("t.csv", t)], + "WITH c AS (SELECT t1.k AS k FROM t t1 JOIN t t2 USING (k)), \ + d(k) AS (SELECT id FROM t INTERSECT SELECT x.k FROM c x JOIN c y USING (k)) \ + SELECT k FROM d WHERE 'D' > 'abc' UNION SELECT COUNT(*) FROM t", + "k\n400\n", + ) +} + +/// The same for a derived table (64 million rows here; 8 s in a release +/// build before the fix above, 0.006 s after). +#[test] +fn derived_table_read_only_under_a_false_where_is_never_computed() -> TestResult { + let t = csv("id,k", (1..=400).map(|i| format!("{i},{}", i % 6))); + assert_fast( + &[("t.csv", t)], + "SELECT COUNT(*) FROM (SELECT t1.k FROM t t1, t t2, t t3) WHERE 'D' > 'abc'", + "COUNT(*)\n0\n", + ) +} + +/// ... and for a CTE read only by a scalar subquery in a CASE branch that is +/// not taken, or in the SELECT list of a query with no rows. +#[test] +fn cte_read_only_by_code_that_never_runs_is_never_computed() -> TestResult { + let t = csv("id,k", (1..=400).map(|i| format!("{i},{}", i % 6))); + assert_fast( + &[("t.csv", t)], + "WITH d AS (SELECT t1.k AS k FROM t t1, t t2, t t3) \ + SELECT CASE WHEN COUNT(*) > 0 THEN 1 ELSE (SELECT COUNT(*) FROM d) END AS x FROM t \ + UNION ALL SELECT (SELECT COUNT(*) FROM d) FROM t WHERE id < 0", + "x\n1\n", + ) +} + +/// The inner table of a join is opened, and its hash index built, when the +/// loop first reaches it: an outer loop with no rows never reads it, so a +/// CTE or derived table there is never computed, as in SQLite (0.0001 s). +/// sqawk opened every table of the join first: `d` is 64 million rows (12 s +/// in a release build, minutes in a debug build). +#[test] +fn cte_joined_to_an_empty_outer_loop_is_never_computed() -> TestResult { + let t = csv("id,k", (1..=400).map(|i| format!("{i},{}", i % 6))); + assert_fast( + &[("t.csv", t)], + "WITH d AS (SELECT t1.k AS k FROM t t1, t t2, t t3) \ + SELECT COUNT(*) AS n FROM t a JOIN d ON a.id = d.k WHERE a.id < 0 \ + UNION ALL SELECT COUNT(*) FROM t a LEFT JOIN d ON a.k = d.k WHERE a.id < 0", + "n\n0\n0\n", + ) +} + +/// A wide table costs time linear in its cells. sqawk copied every column +/// name of the table for each column it read, so a row of n columns cost +/// O(n^2): `SELECT *` over these 3000 columns x 400 rows took about 84 s in +/// a debug build (1000 columns x 1000 rows: 12.8 s in a release build, where +/// `SELECT COUNT(*)` takes 0.02 s). Found by tsq stress (`wide_select`). +#[test] +fn select_star_over_a_wide_table_is_not_quadratic() -> TestResult { + let cols = 3000; + let header: Vec = (0..cols).map(|c| format!("c{c}")).collect(); + let w = csv( + &header.join(","), + (0..400).map(|r| { + (0..cols) + .map(|c| (r * cols + c).to_string()) + .collect::>() + .join(",") + }), + ); + assert_fast(&[("w.csv", w.clone())], "SELECT * FROM w", &w) +} + +/// A one-cell UPDATE written back to a wide file costs time linear in the +/// row's columns: each column read copied every column name of the table, +/// about 80 s for this 36000-column row in a debug build (10000 columns x 20 +/// rows: 20 s in a release build). Found by tsq stress. +#[test] +fn update_write_on_a_wide_file_is_not_quadratic() -> TestResult { + let cols = 36000; + let row = |r: usize, set: Option| { + (0..cols) + .map(|c| match set { + Some(s) if s == c => "-1".to_string(), + _ => (r * cols + c).to_string(), + }) + .collect::>() + .join(",") + }; + let header: Vec = (0..cols).map(|c| format!("c{c}")).collect(); + let header = header.join(","); + let before = csv(&header, (0..3).map(|r| row(r, None))); + let after = csv(&header, (0..3).map(|r| row(r, (r == 1).then_some(7)))); + let dir = create_temp_dir()?; + let path = dir.path().join("w.csv"); + std::fs::write(&path, before)?; + let out = assert_cmd::cargo::cargo_bin_cmd!("sqawk") + .args([ + "-w", + "-s", + &format!("UPDATE w SET c7 = -1 WHERE c0 = {cols}"), + ]) + .arg(&path) + .timeout(Duration::from_secs(20)) + .output()?; + assert!( + out.status.success(), + "sqawk did not finish within 20 s ({})", + out.status + ); + assert_eq!(std::fs::read_to_string(&path)?, after); + Ok(()) +} + +/// A write error names a file outside the working directory by its plain +/// absolute path (`cannot write 'C:\...\ro.csv': file is read-only`). On +/// Windows sqawk printed the verbatim form that `fs::canonicalize` returns, +/// `\\?\C:\...\ro.csv`, a spelling the user never typed. Found by tsq cli +/// under Windows (a `--write` error on a file in another directory). +#[cfg(windows)] +#[test] +fn write_error_path_has_no_verbatim_prefix() -> TestResult { + let data = create_temp_dir()?; + let cwd = create_temp_dir()?; + let file = data.path().join("ro.csv"); + std::fs::write(&file, "id,v\n1,a\n")?; + let mut perms = std::fs::metadata(&file)?.permissions(); + perms.set_readonly(true); + std::fs::set_permissions(&file, perms.clone())?; + let out = assert_cmd::cargo::cargo_bin_cmd!("sqawk") + .current_dir(cwd.path()) + .args(["-w", "-s", "UPDATE ro SET v = 'b'"]) + .arg(&file) + .output()?; + #[allow(clippy::permissions_set_readonly_false)] + perms.set_readonly(false); + std::fs::set_permissions(&file, perms)?; + let real = std::fs::canonicalize(&file)?; + let plain = real.to_string_lossy().replacen(r"\\?\", "", 1); + assert_eq!(out.status.code(), Some(1)); + assert_eq!( + String::from_utf8_lossy(&out.stderr), + format!("sqawk: cannot write '{plain}': file is read-only\n") + ); + Ok(()) +} diff --git a/tests/defects/v1_review_2026_10.rs b/tests/defects/v1_review_2026_10.rs new file mode 100644 index 0000000..0503332 --- /dev/null +++ b/tests/defects/v1_review_2026_10.rs @@ -0,0 +1,1788 @@ +//! Red list: the findings of the 2026-10 v1-readiness review. +//! +//! Every test in this module encodes **correct** behaviour for a finding of +//! that review (a 4,923-query adversarial diff against SQLite plus a +//! release-readiness audit of IO, CLI and REPL behaviour). Each was +//! `#[ignore]`d when written and un-ignored as its defect was fixed; all of +//! them now pass, so the red list is empty again. A few tests (marked in +//! their doc comments) passed when written and pin behaviour the fixes had +//! to keep. Run the red list with: +//! cargo test --test mod defects -- --ignored +//! +//! Expected outputs were computed with SQLite 3.45 (python3 `sqlite3`) unless +//! the doc comment says otherwise. Output NULL is rendered as an EMPTY field. +//! Where sqawk deliberately differs from SQLite the doc comment names the +//! rule being encoded: +//! +//! - numeric-looking text compares numerically everywhere (`=`, `<`, IN list, +//! IN subquery, JOIN, ORDER BY, MIN/MAX), and numbers sort before +//! non-numeric text; DISTINCT / GROUP BY keep `'1'` and `1` apart; +//! - a REAL is rendered as SQLite's `CAST(x AS TEXT)` renders it (`%.15g`, +//! with `.0` on integral values, `Inf` / `-Inf`); +//! - `.tsv` / `.tab` files are tab-separated and every other file is +//! comma-separated unless -F is given; the first line is always the header +//! unless `--no-header` is given; `#` lines are data unless `--comments` is +//! given; +//! - every error is one line `sqawk: ` on stderr with exit status 1. +//! +//! Root-cause locations are hints as of the review (file:line), not contracts. + +use std::fs; +use std::io::{Read, Write}; +use std::path::{Path, PathBuf}; +use std::process::{Command, Output, Stdio}; + +use crate::helpers::{assert_query_on, assert_query_on_fails, create_temp_dir, run_sql, ONE}; + +type TestResult = Result<(), Box>; + +// --------------------------------------------------------------------------- +// Local helpers +// --------------------------------------------------------------------------- + +/// `d(k, n, s)`: three rows, all keys distinct. +const D: (&str, &str) = ("d.csv", "k,n,s\n1,10,x\n2,20,y\n3,30,z\n"); +/// `a(aid)` and `b(aid, v)` for correlated subqueries. +const A: (&str, &str) = ("a.csv", "aid\n1\n2\n"); +const B: (&str, &str) = ("b.csv", "aid,v\n1,5\n1,7\n2,9\n"); +/// `tx(k, n)` for transaction tests. +const TX: &str = "k,n\n1,10\n2,20\n"; + +/// `assert_query_on` over the one-row table `one`. +fn one(sql: &str, expected: &str) -> TestResult { + assert_query_on(&[ONE], sql, expected) +} + +fn sqawk_bin() -> PathBuf { + PathBuf::from(env!("CARGO_BIN_EXE_sqawk")) +} + +/// Run the sqawk binary with `args` in `cwd`, feeding `stdin` (empty when +/// `None`), with HOME and XDG_STATE_HOME pointed at `cwd` so nothing touches +/// the real home directory. +fn run_in(cwd: &Path, args: &[&str], stdin: Option<&[u8]>) -> std::io::Result { + let mut child = Command::new(sqawk_bin()) + .args(args) + .current_dir(cwd) + .env("HOME", cwd) + .env("XDG_STATE_HOME", cwd) + .stdin(Stdio::piped()) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .spawn()?; + { + let mut si = child.stdin.take().unwrap(); + if let Some(input) = stdin { + si.write_all(input)?; + } + } + child.wait_with_output() +} + +/// Write `(name, bytes)` files into `dir`, returning their absolute paths. +fn write_files(dir: &Path, files: &[(&str, &[u8])]) -> std::io::Result> { + let mut out = Vec::new(); + for (name, content) in files { + let p = dir.join(name); + if let Some(parent) = p.parent() { + fs::create_dir_all(parent)?; + } + fs::write(&p, content)?; + out.push(p.to_str().unwrap().to_string()); + } + Ok(out) +} + +/// Write `content` to temp file `file_name`, run the `-s` statements with +/// `extra` args plus `--write`, require success, and return the file bytes. +fn write_back( + file_name: &str, + content: &[u8], + extra: &[&str], + sqls: &[&str], +) -> Result, Box> { + let dir = create_temp_dir()?; + let p = write_files(dir.path(), &[(file_name, content)])?; + let mut args: Vec<&str> = extra.to_vec(); + args.push("--write"); + for s in sqls { + args.push("-s"); + args.push(s); + } + args.push(&p[0]); + let out = run_in(dir.path(), &args, None)?; + assert!( + out.status.success(), + "write run failed\n sql: {sqls:?}\n stderr: {}", + String::from_utf8_lossy(&out.stderr) + ); + Ok(fs::read(&p[0])?) +} + +/// Assert that a `--write` run leaves the file byte-for-byte equal to `expected`. +fn assert_write_back( + file_name: &str, + content: &[u8], + extra: &[&str], + sqls: &[&str], + expected: &[u8], +) -> TestResult { + let actual = write_back(file_name, content, extra, sqls)?; + assert_eq!( + String::from_utf8_lossy(&actual), + String::from_utf8_lossy(expected), + "\n=== sql ===\n{sqls:?}\n=== input ===\n{:?}\n", + String::from_utf8_lossy(content) + ); + assert_eq!(actual, expected, "bytes differ (invalid UTF-8 region)"); + Ok(()) +} + +/// Run `sqls` with `--write` over `file_name` and require that the run FAILS +/// and leaves the file byte-identical. +fn assert_write_refused(file_name: &str, content: &[u8], sqls: &[&str]) -> TestResult { + let dir = create_temp_dir()?; + let p = write_files(dir.path(), &[(file_name, content)])?; + let mut args = vec!["--write"]; + for s in sqls { + args.push("-s"); + args.push(s); + } + args.push(&p[0]); + let out = run_in(dir.path(), &args, None)?; + assert!( + !out.status.success(), + "run unexpectedly succeeded\n sql: {sqls:?}" + ); + assert_eq!(fs::read(&p[0])?, content, "file changed\n sql: {sqls:?}"); + Ok(()) +} + +/// The CLI error contract: exit status 1, stderr is exactly one line starting +/// `sqawk: `, no "Caused by" chain, no panic. +fn assert_clean_error(out: &Output, must_contain: &[&str]) { + let stderr = String::from_utf8_lossy(&out.stderr); + assert_eq!(out.status.code(), Some(1), "exit status; stderr:\n{stderr}"); + assert!(!stderr.contains("panicked"), "panicked:\n{stderr}"); + assert!( + !stderr.contains("Caused by"), + "error chain printed:\n{stderr}" + ); + assert_eq!( + stderr.lines().count(), + 1, + "stderr is not one line:\n{stderr}" + ); + assert!( + stderr.starts_with("sqawk: "), + "stderr lacks the `sqawk: ` prefix:\n{stderr}" + ); + for s in must_contain { + assert!(stderr.contains(s), "stderr lacks {s:?}:\n{stderr}"); + } +} + +/// Run sqawk with `-s` statements over `(name, content)` files in a temp dir. +fn run_sqls( + files: &[(&str, &str)], + extra: &[&str], + sqls: &[&str], +) -> Result> { + let dir = create_temp_dir()?; + let bytes: Vec<(&str, &[u8])> = files.iter().map(|(n, c)| (*n, c.as_bytes())).collect(); + let paths = write_files(dir.path(), &bytes)?; + let mut args: Vec<&str> = extra.to_vec(); + for s in sqls { + args.push("-s"); + args.push(s); + } + for p in &paths { + args.push(p); + } + Ok(run_in(dir.path(), &args, None)?) +} + +/// Run the `-s` statements over `files` and require exactly `expected` on +/// stdout (byte-exact apart from a trailing newline). +fn assert_sqls( + files: &[(&str, &str)], + extra: &[&str], + sqls: &[&str], + expected: &str, +) -> TestResult { + let out = run_sqls(files, extra, sqls)?; + let stdout = String::from_utf8_lossy(&out.stdout); + assert!( + out.status.success(), + "run failed\n sql: {sqls:?}\n stderr: {}", + String::from_utf8_lossy(&out.stderr) + ); + assert_eq!( + stdout.trim_end_matches('\n'), + expected, + "\n=== sql ===\n{sqls:?}\n" + ); + Ok(()) +} + +/// Require that the run fails without crashing: nonzero exit status that is +/// not a signal, no panic, no stack overflow, a non-empty message. +fn assert_fails_cleanly(out: &Output) { + let stderr = String::from_utf8_lossy(&out.stderr); + let code = out.status.code(); + assert!( + matches!(code, Some(c) if c != 0 && c != 101), + "exit status {:?}: expected a clean failure (nonzero, not a signal or panic)\nstderr: {}", + out.status, + &stderr[..stderr.len().min(500)] + ); + assert!(!stderr.contains("panicked"), "panicked"); + assert!(!stderr.contains("overflow"), "stack overflow"); + assert!(!stderr.trim().is_empty(), "no error message"); +} + +/// True when file permissions are enforced for this user (false for root). +#[cfg(unix)] +fn perms_enforced(dir: &Path) -> bool { + let p = dir.join(".perm_probe"); + fs::write(&p, "x").unwrap(); + let mut perm = fs::metadata(&p).unwrap().permissions(); + std::os::unix::fs::PermissionsExt::set_mode(&mut perm, 0o000); + fs::set_permissions(&p, perm).unwrap(); + let enforced = fs::File::open(&p).is_err(); + fs::remove_file(&p).unwrap(); + enforced +} + +#[cfg(unix)] +fn chmod(p: &Path, mode: u32) { + let mut perm = fs::metadata(p).unwrap().permissions(); + std::os::unix::fs::PermissionsExt::set_mode(&mut perm, mode); + fs::set_permissions(p, perm).unwrap(); +} + +/// Generate `SELECT t1 t2 ... AS v FROM one` with `n` terms. +fn chain(n: usize, sep: &str, term: &str) -> String { + format!("SELECT {} AS v FROM one", vec![term; n].join(sep)) +} + +/// Run `sql` over the one-row table and return the process output. +/// +/// The SQL goes in through `-f -` on stdin, not `-s`: the deep-expression +/// tests use statements of 20-70 KB, past Windows' 32,767-character command +/// line limit. +fn run_one(sql: &str) -> Result> { + let dir = create_temp_dir()?; + let paths = write_files(dir.path(), &[(ONE.0, ONE.1.as_bytes())])?; + Ok(run_in( + dir.path(), + &["-f", "-", &paths[0]], + Some(sql.as_bytes()), + )?) +} + +/// Run a piped (non-tty) REPL over `files` in a temp dir with `script` on +/// stdin. Returns the output and the temp dir (for inspecting files). +fn run_repl( + files: &[(&str, &str)], + script: &str, +) -> Result<(Output, tempfile::TempDir), Box> { + let dir = create_temp_dir()?; + let bytes: Vec<(&str, &[u8])> = files.iter().map(|(n, c)| (*n, c.as_bytes())).collect(); + write_files(dir.path(), &bytes)?; + let mut args = vec!["-i"]; + for (n, _) in files { + args.push(n); + } + let out = run_in(dir.path(), &args, Some(script.as_bytes()))?; + Ok((out, dir)) +} + +/// Table names listed by `.tables ` in a piped REPL. +fn repl_tables(pattern: &str) -> Result, Box> { + let files = [ + ("emp.csv", "a\n1\n"), + ("exa.csv", "a\n1\n"), + ("Ebig.csv", "a\n1\n"), + ("rw.csv", "a\n1\n"), + ]; + let (out, _dir) = run_repl(&files, &format!(".tables {pattern}\n"))?; + let stdout = String::from_utf8_lossy(&out.stdout); + let names = ["emp", "exa", "Ebig", "rw"]; + let mut listed: Vec = stdout + .lines() + .map(str::trim) + .filter(|l| names.contains(l)) + .map(String::from) + .collect(); + listed.sort(); + Ok(listed) +} + +// =========================================================================== +// A. Semantics +// =========================================================================== + +// --------------------------------------------------------------------------- +// A1. Name resolution: unknown qualifiers silently resolve +// (`NameCtx::lookup` / `resolve_qualified`, compiler.rs ~321/~342: a +// single-source query accepts any qualifier) +// --------------------------------------------------------------------------- + +/// `zz` names no table in FROM, yet `zz.k` resolves to `d.k`. +#[test] +fn unknown_qualifier_is_error() -> TestResult { + assert_query_on_fails(&[D], "SELECT zz.k AS v FROM d", "no such column: zz.k") +} + +/// An UPDATE whose WHERE names an unknown qualifier updates rows and writes +/// them back. It must fail and leave the file unchanged. +#[test] +fn unknown_qualifier_in_update_is_error() -> TestResult { + assert_write_refused( + "d.csv", + D.1.as_bytes(), + &["UPDATE d SET n = 0 WHERE zz.k = 1"], + ) +} + +/// `zz.*` with no table `zz` expands to `d.*` (star expansion, compiler.rs ~897). +#[test] +fn unknown_qualifier_star_is_error() -> TestResult { + assert_query_on_fails(&[D], "SELECT zz.* FROM d", "zz") +} + +/// A derived table that references an outer column is materialized once, +/// without the correlation (`materialize_in_factor`, vm/mod.rs ~409), so every +/// row gets max(v) over all of `b` (9, 9). It must be rejected. +#[test] +fn correlated_derived_table_is_error() -> TestResult { + let out = run_sqls( + &[A, B], + &[], + &["SELECT aid, (SELECT max(v) FROM (SELECT b.v FROM b WHERE b.aid = a.aid) q) AS m FROM a"], + )?; + assert_fails_cleanly(&out); + Ok(()) +} + +/// Guard (passes today): an alias on the UPDATE target resolves. The +/// qualifier fix must keep it working (pass the alias into `Src.refname`, +/// compiler_dml.rs ~210). +#[test] +fn aliased_update_target_resolves() -> TestResult { + assert_write_back( + "d.csv", + D.1.as_bytes(), + &[], + &["UPDATE d AS x SET n = 1 WHERE x.k = 1"], + b"k,n,s\n1,1,x\n2,20,y\n3,30,z\n", + ) +} + +/// A schema qualifier other than `main` is silently ignored: `foo.d` reads `d`. +#[test] +fn unknown_schema_is_error() -> TestResult { + assert_query_on_fails(&[D], "SELECT * FROM foo.d", "foo") +} + +/// `UPDATE foo.d` updates `d` and writes it back. It must fail, file unchanged. +#[test] +fn unknown_schema_in_update_is_error() -> TestResult { + assert_write_refused("d.csv", D.1.as_bytes(), &["UPDATE foo.d SET n = 5"]) +} + +/// Guard (passes today): SQLite's `main` schema qualifier is accepted. +#[test] +fn main_schema_qualifier_works() -> TestResult { + assert_query_on(&[D], "SELECT k FROM main.d AS x WHERE x.k = 1", "k\n1") +} + +// --------------------------------------------------------------------------- +// A2. Operator precedence differs from SQLite (Hive dialect precedence in +// sqlparser; plan: SqawkDialect with SQLite levels, compounds left-deep) +// --------------------------------------------------------------------------- + +/// `IS NOT DISTINCT FROM` parses its RHS as `0 AND 0`; SQLite binds it at the +/// `=` level: `(0 IS NOT DISTINCT FROM 0) AND 0` = 0. +#[test] +fn is_not_distinct_from_binds_tighter_than_and() -> TestResult { + one( + "SELECT 0 IS NOT DISTINCT FROM 0 AND 0 AS v FROM one", + "v\n0", + ) +} + +/// Same defect in a WHERE: parsed as `k IS NOT DISTINCT FROM (2 AND n > 0)`, +/// which selects k = 1 instead of k = 2. +#[test] +fn is_not_distinct_from_in_where() -> TestResult { + assert_query_on( + &[D], + "SELECT k FROM d WHERE k IS NOT DISTINCT FROM 2 AND n > 0", + "k\n2", + ) +} + +/// LIKE and `=` share one precedence level in SQLite (left-to-right): +/// `('abc' LIKE 'A%') = 1` = 1. +#[test] +fn like_and_equals_same_level() -> TestResult { + one("SELECT 'abc' LIKE 'A%' = 1 AS v FROM one", "v\n1") +} + +/// `<` binds tighter than `=` in SQLite: `2 = (1 < 2)` = 0. +#[test] +fn less_than_binds_tighter_than_equals() -> TestResult { + one("SELECT 2 = 1 < 2 AS v FROM one", "v\n0") +} + +/// `||` binds tighter than `*` in SQLite: `2 * (3 || 4)` = 68. +#[test] +fn concat_binds_tighter_than_multiply() -> TestResult { + one("SELECT 2 * 3 || 4 AS v FROM one", "v\n68") +} + +/// Compound operators have equal precedence, left to right, in SQLite: +/// `(1 UNION ALL 2) INTERSECT 2` = {2}. sqawk binds INTERSECT tighter. +#[test] +fn compound_operators_left_to_right() -> TestResult { + one( + "SELECT 1 AS v UNION ALL SELECT 2 INTERSECT SELECT 2", + "v\n2", + ) +} + +/// Guard (passes today): `(1 INTERSECT 1) UNION 2` = {1, 2}. +#[test] +fn compound_intersect_then_union() -> TestResult { + one("SELECT 1 AS v INTERSECT SELECT 1 UNION SELECT 2", "v\n1\n2") +} + +// --------------------------------------------------------------------------- +// A3. Comparison rule unification. Rule (deliberate difference from SQLite): +// numeric-looking text compares numerically everywhere; numbers sort before +// non-numeric text. (IN-subquery set: compiler_subquery.rs ~136, engine +// `Found`/`row_key`; `sql_compare` engine ~2621; `compare_values` ~500) +// --------------------------------------------------------------------------- + +/// `'9' IN (9)` and `'9' = (SELECT 9)` are 1 but `'9' IN (SELECT 9)` is 0: +/// the IN-subquery set compares by exact key. +#[test] +fn in_subquery_numeric_text_matches_number() -> TestResult { + one( + "SELECT '9' IN (SELECT 9) AS a, '9' IN (9) AS b, '9' = (SELECT 9) AS c FROM one", + "a,b,c\n1,1,1", + ) +} + +/// An IN subquery must match the same rows as the equivalent JOIN. `tk.k` +/// holds numeric-looking text (`007`, `9.0`, `1e1`); the JOIN matches all +/// four rows, the IN subquery only 8. +#[test] +fn in_subquery_agrees_with_join() -> TestResult { + let files = [ + ("na.csv", "n\n7\n8\n9\n10\n"), + ("tk.csv", "k\n007\n8\n9.0\n1e1\n"), + ]; + assert_query_on( + &files, + "SELECT na.n AS n FROM na JOIN tk ON na.n = tk.k ORDER BY 1", + "n\n7\n8\n9\n10", + )?; + assert_query_on( + &files, + "SELECT n FROM na WHERE n IN (SELECT k FROM tk) ORDER BY n", + "n\n7\n8\n9\n10", + ) +} + +/// Two numeric-looking texts compare numerically, as ORDER BY and MIN/MAX +/// already order them (`'9'` before `'10'`). `<` compares them lexically. +#[test] +fn numeric_text_less_than_is_numeric() -> TestResult { + one( + "SELECT '10' < '9' AS a, '9' < '10' AS b, '010' < '9' AS c FROM one", + "a,b,c\n0,1,0", + ) +} + +/// Numbers sort before non-numeric text in ORDER BY; `<` must agree: +/// `'-x' < 10` is 0 and `10 < 'abc'` is 1. +#[test] +fn non_numeric_text_vs_number_order() -> TestResult { + one( + "SELECT '-x' < 10 AS a, 10 < 'abc' AS b FROM one", + "a,b\n0,1", + ) +} + +/// ORDER BY puts `9, 10` before `-x, abc`, but `s < '9'` selects `-x`. +/// Under one total order no row is below 9. +#[test] +fn less_than_agrees_with_order_by() -> TestResult { + let st = [("st.csv", "s\n10\n9\nabc\n-x\n")]; + assert_query_on(&st, "SELECT s FROM st ORDER BY s", "s\n9\n10\n-x\nabc")?; + assert_query_on(&st, "SELECT s FROM st WHERE s < '9' ORDER BY s", "s") +} + +/// Guard (passes today, matches SQLite): DISTINCT and GROUP BY keep the text +/// `'1'` and the integer `1` apart. +#[test] +fn distinct_and_group_by_keep_text_and_integer_apart() -> TestResult { + one( + "SELECT DISTINCT v FROM (SELECT '1' AS v UNION ALL SELECT 1 UNION ALL SELECT 1) AS q", + "v\n1\n1", + )?; + one( + "SELECT typeof(v) AS t, count(*) AS c FROM \ + (SELECT '1' AS v UNION ALL SELECT 1 UNION ALL SELECT '1') AS q GROUP BY v ORDER BY t", + "t,c\ninteger,1\ntext,2", + ) +} + +// --------------------------------------------------------------------------- +// A4. Aggregate of outer-only columns in a subquery +// (`compile_select_grouped`, compiler_aggregate.rs ~228) +// --------------------------------------------------------------------------- + +/// `MIN(d.n)` inside a subquery over `b` aggregates only outer columns; sqawk +/// returns NULL for every row. It must be a clean error. +#[test] +fn outer_only_aggregate_in_subquery_is_error() -> TestResult { + let out = run_sqls( + &[D, B], + &[], + &["SELECT k, (SELECT MIN(d.n) FROM b WHERE b.v >= d.n) AS m FROM d"], + )?; + assert_fails_cleanly(&out); + Ok(()) +} + +// --------------------------------------------------------------------------- +// A5. Deep expressions overflow the stack (recursive parse/compile/eval). +// Limits as SQLite: expression depth 1000, 500 compound terms. +// --------------------------------------------------------------------------- + +/// 10,000-term `1+1+...`: SIGABRT stack overflow. Must be a clean error. +#[test] +fn deep_plus_chain_is_clean_error() -> TestResult { + assert_fails_cleanly(&run_one(&chain(10_000, "+", "1"))?); + Ok(()) +} + +/// 7,000-term OR chain: stack overflow. +#[test] +fn deep_or_chain_is_clean_error() -> TestResult { + assert_fails_cleanly(&run_one(&chain(7_000, " OR ", "id = 2"))?); + Ok(()) +} + +/// 10,000-term AND chain: stack overflow. +#[test] +fn deep_and_chain_is_clean_error() -> TestResult { + assert_fails_cleanly(&run_one(&chain(10_000, " AND ", "id = 1"))?); + Ok(()) +} + +/// 5,000-term `||` chain: stack overflow. +#[test] +fn deep_concat_chain_is_clean_error() -> TestResult { + assert_fails_cleanly(&run_one(&chain(5_000, " || ", "'a'"))?); + Ok(()) +} + +/// 3,000 UNION ALL arms: stack overflow. +#[test] +fn many_union_arms_is_clean_error() -> TestResult { + let sql = (0..3000) + .map(|i| format!("SELECT {i} AS v")) + .collect::>() + .join(" UNION ALL "); + assert_fails_cleanly(&run_one(&sql)?); + Ok(()) +} + +/// A 900-term `1+...` chain is below SQLite's depth limit and must work; the +/// debug build overflows its stack today. +#[test] +fn plus_chain_of_900_works() -> TestResult { + one(&chain(900, "+", "1"), "v\n900") +} + +/// Guard (passes today): 400 UNION ALL arms (below the 500-term limit) work. +#[test] +fn union_of_400_arms_works() -> TestResult { + let sql = (0..400) + .map(|i| format!("SELECT {i} AS v")) + .collect::>() + .join(" UNION ALL "); + let expected = std::iter::once("v".to_string()) + .chain((0..400).map(|i| i.to_string())) + .collect::>() + .join("\n"); + one(&sql, &expected) +} + +// --------------------------------------------------------------------------- +// A6. Result column names (SQLite rules: a column reference is named by its +// bare column name, `*` expands to bare names, any other unaliased +// expression is named by its SQL text as written). `output_name` +// compiler.rs ~936, `get_column_name_from_expr` ~2004, `star_name` ~850. +// --------------------------------------------------------------------------- + +const U: (&str, &str) = ("u.csv", "id,name\n1,ann\n"); +const O: (&str, &str) = ("o.csv", "id,uid\n7,1\n"); +const T: (&str, &str) = ("t.csv", "n,name\n5,ab\n"); + +/// In a join, `u.name, o.id` are headed `u.name,o.id`; SQLite: `name,id`. +#[test] +fn qualified_column_header_is_bare_name() -> TestResult { + assert_query_on( + &[U, O], + "SELECT u.name, o.id FROM u JOIN o ON u.id = o.uid", + "name,id\nann,7", + ) +} + +/// `*` over a join is headed `u.id,u.name,o.id,o.uid`; SQLite uses bare +/// names, duplicates included. +#[test] +fn star_over_join_header_is_bare_names() -> TestResult { + assert_query_on(&[U, O], "SELECT * FROM u, o", "id,name,id,uid\n1,ann,7,1") +} + +/// Unaliased expressions are headed `expr`; SQLite uses the text as written. +#[test] +fn expression_header_is_sql_text() -> TestResult { + assert_query_on(&[T], "SELECT n*2 FROM t", "n*2\n10")?; + assert_query_on(&[T], "SELECT upper(name)||1 FROM t", "upper(name)||1\nAB1")?; + assert_query_on(&[T], "SELECT n + 1 FROM t", "n + 1\n6") +} + +/// `COUNT(*)` is headed `COUNT`; SQLite: `COUNT(*)`. +#[test] +fn count_star_header_is_sql_text() -> TestResult { + assert_query_on(&[T], "SELECT COUNT(*) FROM t", "COUNT(*)\n1") +} + +// --------------------------------------------------------------------------- +// A7. REAL rendering and summation. Rendering = SQLite `CAST(x AS TEXT)`: +// `%.15g`, `.0` on integral values, `Inf`/`-Inf` (Value Display, table.rs +// ~158; `arith::text_of` ~290; share `c_general`, func.rs ~620). SUM/TOTAL/ +// AVG use Neumaier summation as SQLite does (agg.rs `Sum::Real`). +// --------------------------------------------------------------------------- + +/// Printed as 301 digits instead of `1.5e+300`. +#[test] +fn large_real_uses_exponent() -> TestResult { + one("SELECT 1e300 * 1.5 AS v FROM one", "v\n1.5e+300") +} + +/// Printed as `0.000...015` instead of `1.5e-300`. +#[test] +fn tiny_real_uses_exponent() -> TestResult { + one("SELECT 1.5e-300 AS v FROM one", "v\n1.5e-300") +} + +/// Printed as `0.30000000000000004` instead of `0.3`. +#[test] +fn real_prints_15_significant_digits() -> TestResult { + one("SELECT 0.1 + 0.2 AS v FROM one", "v\n0.3") +} + +/// Integer overflow to real prints `9223372036854776000`; SQLite +/// `9.22337203685478e+18`. +#[test] +fn overflowed_integer_prints_as_real() -> TestResult { + one( + "SELECT 9223372036854775807 + 1 AS v FROM one", + "v\n9.22337203685478e+18", + ) +} + +/// An integral real prints `2`; SQLite renders `2.0`. +#[test] +fn integral_real_prints_point_zero() -> TestResult { + one("SELECT 2.0 * 1 AS v FROM one", "v\n2.0") +} + +/// Infinity prints `inf` / `-inf`; SQLite `Inf` / `-Inf`. +#[test] +fn infinity_prints_like_sqlite() -> TestResult { + one( + "SELECT 1e308 * 10 AS v, -1e308 * 10 AS w FROM one", + "v,w\nInf,-Inf", + ) +} + +/// `ROUND(x, N)` with negative N rounds to tens/hundreds (1200); SQLite +/// treats negative N as 0: `ROUND(1234.5, -2)` = 1235.0 (func.rs ~666). +#[test] +fn round_negative_digits_is_zero() -> TestResult { + one("SELECT ROUND(1234.5, -2) AS v FROM one", "v\n1235.0") +} + +/// Naive summation: SUM over 17.07, 10.51, 23.41, 29.99, 22.82 is +/// 103.79999999999998, so `SUM(x) = 103.8` is 0. SQLite (Neumaier) gives +/// exactly 103.8. +#[test] +fn sum_is_compensated() -> TestResult { + let m = [("m.csv", "x\n17.07\n10.51\n23.41\n29.99\n22.82\n")]; + assert_query_on( + &m, + "SELECT SUM(x) = 103.8 AS e, TOTAL(x) = 103.8 AS t, SUM(x) AS s, AVG(x) AS a FROM m", + "e,t,s,a\n1,1,103.8,20.76", + ) +} + +/// SUM of 0.1, 0.2, 0.3 prints `0.6000000000000001`; SQLite `0.6`, and +/// `SUM(x) = 0.6` is 1. +#[test] +fn sum_of_tenths() -> TestResult { + let p = [("p.csv", "x\n0.1\n0.2\n0.3\n")]; + assert_query_on( + &p, + "SELECT SUM(x) AS s, SUM(x) = 0.6 AS e FROM p", + "s,e\n0.6,1", + ) +} + +// =========================================================================== +// B. IO / data safety +// =========================================================================== + +// --------------------------------------------------------------------------- +// B8. Verbatim writeback: untouched rows must be written byte-for-byte as +// read (record each row's raw bytes at load; csv_handler.rs / table.rs) +// --------------------------------------------------------------------------- + +/// Unnecessary quotes on an untouched row are dropped (`"abc"` -> `abc`). +#[test] +fn writeback_keeps_unnecessary_quotes() -> TestResult { + assert_write_back( + "q.csv", + b"k,s\n1,\"abc\"\n2,x\n", + &[], + &["UPDATE q SET s = 'y' WHERE k = 2"], + b"k,s\n1,\"abc\"\n2,y\n", + ) +} + +/// An unquoted trailing space on an untouched row gets quoted (`"trail "`). +#[test] +fn writeback_keeps_trailing_space_unquoted() -> TestResult { + assert_write_back( + "q.csv", + b"k,s\n1,trail \n2,x\n", + &[], + &["UPDATE q SET s = 'y' WHERE k = 2"], + b"k,s\n1,trail \n2,y\n", + ) +} + +/// A TSV field `a"b` on an untouched row is rewritten as `"a""b"`. +#[test] +fn writeback_keeps_tsv_quote_char() -> TestResult { + assert_write_back( + "t.tsv", + b"k\ts\n1\ta\"b\n2\tz\n", + &[], + &["UPDATE t SET s = 'y' WHERE k = 2"], + b"k\ts\n1\ta\"b\n2\ty\n", + ) +} + +/// The UTF-8 BOM is dropped on writeback. +#[test] +fn writeback_keeps_bom() -> TestResult { + assert_write_back( + "bom.csv", + b"\xef\xbb\xbfk,s\n1,a\n2,b\n", + &[], + &["UPDATE bom SET s = 'y' WHERE k = 2"], + b"\xef\xbb\xbfk,s\n1,a\n2,y\n", + ) +} + +/// A short (ragged) untouched row is padded with a trailing comma. +#[test] +fn writeback_keeps_short_row() -> TestResult { + assert_write_back( + "rag.csv", + b"k,s,t\n1,a\n2,b,c\n", + &[], + &["UPDATE rag SET s = 'y' WHERE k = 2"], + b"k,s,t\n1,a\n2,y,c\n", + ) +} + +/// Invalid UTF-8 in an untouched row is replaced by U+FFFD. +#[test] +fn writeback_keeps_invalid_utf8() -> TestResult { + assert_write_back( + "bad.csv", + b"k,s\n1,\xff\xfe\n2,b\n", + &[], + &["UPDATE bad SET s = 'y' WHERE k = 2"], + b"k,s\n1,\xff\xfe\n2,y\n", + ) +} + +/// The original last line (no final newline, quoted) is rewritten when a row +/// is appended. It must stay `2,"b"`, with the new row on its own line. +#[test] +fn writeback_insert_after_unterminated_last_line() -> TestResult { + assert_write_back( + "nq.csv", + b"k,s\n1,a\n2,\"b\"", + &[], + &["INSERT INTO nq VALUES (3, 'c')"], + b"k,s\n1,a\n2,\"b\"\n3,c\n", + ) +} + +/// A no-op UPDATE over every row leaves the file byte-identical. +#[test] +fn writeback_noop_update_is_identity() -> TestResult { + let content = b"k,s\n1,\"abc\"\n2,trail \n3,x\n"; + assert_write_back("q.csv", content, &[], &["UPDATE q SET k = k"], content) +} + +// --------------------------------------------------------------------------- +// B9. Unclosed quote is silently accepted (the csv crate yields the rest of +// the file as one field) +// --------------------------------------------------------------------------- + +/// `1,"2\n3,4\n` loads as one row with b = `2\n3,4\n`. Must be an error +/// naming the line where the quoted field starts. +#[test] +fn unclosed_quote_is_error() -> TestResult { + let out = run_sqls( + &[("uq.csv", "a,b\n1,\"2\n3,4\n")], + &[], + &["SELECT * FROM uq"], + )?; + assert_clean_error(&out, &["unterminated", "line 2"]); + Ok(()) +} + +// --------------------------------------------------------------------------- +// B10. Duplicate / empty column names (csv_handler.rs ~126; compiler_ddl.rs) +// --------------------------------------------------------------------------- + +fn assert_load_fails(name: &str, content: &str) -> TestResult { + let out = run_sqls(&[(name, content)], &[], &["SELECT 1 AS v"])?; + assert_fails_cleanly(&out); + Ok(()) +} + +/// Header `a,a` loads; both columns read as the first. +#[test] +fn duplicate_header_is_error() -> TestResult { + assert_load_fails("dh.csv", "a,a\n1,2\n") +} + +/// Header `a,A` loads; column names are case-insensitive, so it is a duplicate. +#[test] +fn case_insensitive_duplicate_header_is_error() -> TestResult { + assert_load_fails("dhc.csv", "a,A\n1,2\n") +} + +/// Header `a,,c` loads with an empty column name. +#[test] +fn empty_header_name_is_error() -> TestResult { + assert_load_fails("eh.csv", "a,,c\n1,2,3\n") +} + +/// `ALTER TABLE al ADD COLUMN a INT` adds a second column `a`. +#[test] +fn alter_add_existing_column_is_error() -> TestResult { + let out = run_sqls( + &[("al.csv", "a,b\n1,2\n")], + &[], + &["ALTER TABLE al ADD COLUMN a INT"], + )?; + assert_fails_cleanly(&out); + Ok(()) +} + +/// `CREATE TABLE ct (x INT, x INT)` creates two columns named `x`. +#[test] +fn create_table_duplicate_column_is_error() -> TestResult { + assert_fails_cleanly(&run_one("CREATE TABLE ct (x INT, x INT)")?); + Ok(()) +} + +/// `CREATE TABLE ct (x INT, X INT)`: a case-insensitive duplicate. +#[test] +fn create_table_case_insensitive_duplicate_is_error() -> TestResult { + assert_fails_cleanly(&run_one("CREATE TABLE ct (x INT, X INT)")?); + Ok(()) +} + +// --------------------------------------------------------------------------- +// B11. Table specs (`file_handler.rs` ~286, `main.rs` ~110): a `name=` prefix +// is a table name only when it is identifier-like; paths are canonical +// --------------------------------------------------------------------------- + +/// `/k=v/e.csv` is split at `=` into table `/k` and file `v/e.csv` +/// ("File not found"). It is a path; the table is `e`. +#[test] +fn equals_in_directory_is_path() -> TestResult { + let dir = create_temp_dir()?; + let p = write_files(dir.path(), &[("k=v/e.csv", b"a\n1\n")])?; + let (stdout, stderr, ok) = run_sql("SELECT a FROM e", &[&p[0]], &[])?; + assert!(ok, "load failed: {stderr}"); + assert_eq!(stdout.trim_end(), "a\n1"); + Ok(()) +} + +/// `=al.csv` (empty table name) is accepted. +#[test] +fn empty_table_name_is_error() -> TestResult { + let dir = create_temp_dir()?; + write_files(dir.path(), &[("al.csv", b"a\n1\n")])?; + let out = run_in(dir.path(), &["-s", "SELECT 1 AS v", "=al.csv"], None)?; + assert_fails_cleanly(&out); + Ok(()) +} + +/// The same file loaded under two names is accepted; with --write the two +/// copies would overwrite each other. Must be an error. +#[test] +fn same_file_twice_is_error() -> TestResult { + let dir = create_temp_dir()?; + write_files(dir.path(), &[("al.csv", b"a\n1\n")])?; + let out = run_in( + dir.path(), + &["-s", "SELECT a FROM x1", "x1=al.csv", "x2=./al.csv"], + None, + )?; + assert_fails_cleanly(&out); + Ok(()) +} + +/// REPL `.cd ..` then `.save` writes `s.csv` into the NEW directory (the path +/// is stored relative). It must write the file that was loaded. +#[test] +fn repl_save_after_cd_writes_original_file() -> TestResult { + let dir = create_temp_dir()?; + let sub = dir.path().join("sub"); + write_files(dir.path(), &[("sub/s.csv", b"a\n1\n")])?; + let out = run_in( + &sub, + &["-i", "s.csv"], + Some(b"INSERT INTO s VALUES (9);\n.cd ..\n.save\n"), + )?; + assert!( + out.status.success(), + "{}", + String::from_utf8_lossy(&out.stderr) + ); + assert_eq!(fs::read_to_string(sub.join("s.csv"))?, "a\n1\n9\n"); + assert!( + !dir.path().join("s.csv").exists(), + ".save wrote a new file in the cd target" + ); + Ok(()) +} + +// --------------------------------------------------------------------------- +// B12. Silently ignored inputs (compiler_check.rs ~357; database.rs ~147-183) +// --------------------------------------------------------------------------- + +/// `STORED AS PARQUET` is ignored: a CSV is written to `p.pq`. +#[test] +fn stored_as_parquet_is_error() -> TestResult { + let dir = create_temp_dir()?; + write_files(dir.path(), &[("al.csv", b"a\n1\n")])?; + let out = run_in( + dir.path(), + &[ + "-w", + "-s", + "CREATE TABLE p (a INT) LOCATION 'p.pq' STORED AS PARQUET", + "-s", + "INSERT INTO p VALUES (1)", + "al.csv", + ], + None, + )?; + assert_fails_cleanly(&out); + assert!(!dir.path().join("p.pq").exists(), "p.pq was written"); + Ok(()) +} + +fn tabledef_fails(def: &str, extra: &[&str]) -> TestResult { + let mut args = vec![def]; + args.extend_from_slice(extra); + let out = run_sqls(&[("al.csv", "a,b\n1,2\n")], &args, &["SELECT * FROM al"])?; + assert_fails_cleanly(&out); + Ok(()) +} + +/// `--tabledef=bad` (no `:`) is ignored. +#[test] +fn tabledef_without_colon_is_error() -> TestResult { + tabledef_fails("--tabledef=bad", &[]) +} + +/// `--tabledef=nosuch:a,b` names no loaded table and is ignored. +#[test] +fn tabledef_unknown_table_is_error() -> TestResult { + tabledef_fails("--tabledef=nosuch:a,b", &[]) +} + +/// `--tabledef=al:x,` creates a column with an empty name. +#[test] +fn tabledef_empty_column_is_error() -> TestResult { + tabledef_fails("--tabledef=al:x,", &["--no-header"]) +} + +// --------------------------------------------------------------------------- +// B13. Writeback edge cases (sql_executor.rs ~123, csv_handler.rs save ~320) +// --------------------------------------------------------------------------- + +/// A read-only (0444) file is replaced by the atomic rename. --write must +/// refuse up front and write nothing, including other modified files. +#[cfg(unix)] +#[test] +fn write_to_read_only_file_is_refused() -> TestResult { + let dir = create_temp_dir()?; + if !perms_enforced(dir.path()) { + return Ok(()); + } + let p = write_files( + dir.path(), + &[("rw.csv", b"a,b\n1,2\n"), ("ro.csv", b"a,b\n1,2\n")], + )?; + chmod(Path::new(&p[1]), 0o444); + let out = run_in( + dir.path(), + &[ + "-w", + "-s", + "INSERT INTO rw VALUES (3, 4)", + "-s", + "INSERT INTO ro VALUES (5, 6)", + "rw.csv", + "ro.csv", + ], + None, + )?; + chmod(Path::new(&p[1]), 0o644); + assert_clean_error(&out, &["ro.csv"]); + assert_eq!(fs::read_to_string(&p[0])?, "a,b\n1,2\n", "rw.csv written"); + assert_eq!(fs::read_to_string(&p[1])?, "a,b\n1,2\n", "ro.csv written"); + Ok(()) +} + +/// A scratch table (CREATE TABLE without LOCATION) makes --write fail with +/// "has no associated file path", and the real modified file is not written. +/// The scratch table must be skipped. +#[test] +fn write_skips_scratch_table() -> TestResult { + assert_write_back( + "rw.csv", + b"a,b\n1,2\n", + &[], + &[ + "CREATE TABLE sc (x INT)", + "INSERT INTO sc VALUES (1)", + "INSERT INTO rw VALUES (3, 4)", + ], + b"a,b\n1,2\n3,4\n", + ) +} + +/// DROP of a modified table makes --write fail with "Table 'rw' not found". +/// The dropped table is not written; the run succeeds. +#[test] +fn write_after_drop_of_modified_table() -> TestResult { + assert_write_back( + "rw.csv", + b"a,b\n1,2\n", + &[], + &["INSERT INTO rw VALUES (3, 4)", "DROP TABLE rw"], + b"a,b\n1,2\n", + ) +} + +// --------------------------------------------------------------------------- +// B14. File-type rules (file_handler.rs ~192-215): `.tsv`/`.tab` are tab, +// everything else comma; first line always the header; `#` is data unless +// `--comments` +// --------------------------------------------------------------------------- + +/// A comma-separated `.txt` file is split on tabs (one column `a,b`). +#[test] +fn txt_file_is_comma_separated() -> TestResult { + assert_query_on(&[("c.txt", "a,b\n1,2\n")], "SELECT b FROM c", "b\n2") +} + +/// A numeric first line in a `.tsv` is guessed to be data (columns a, b). +/// It is the header. +#[test] +fn tsv_first_line_is_header() -> TestResult { + assert_query_on( + &[("num.tsv", "1\t2\n3\t4\n")], + "SELECT * FROM num", + "1,2\n3,4", + ) +} + +/// Same for a file without an extension. +#[test] +fn extensionless_first_line_is_header() -> TestResult { + assert_query_on(&[("numx", "1,2\n3,4\n")], "SELECT * FROM numx", "1,2\n3,4") +} + +/// `#` lines in a `.tsv` are skipped as comments by default; they are data. +#[test] +fn hash_line_is_data_by_default() -> TestResult { + assert_query_on( + &[("h.tsv", "a\tb\n#x\t1\n2\t3\n")], + "SELECT * FROM h", + "a,b\n#x,1\n2,3", + ) +} + +/// `--comments` is rejected as an unknown flag. It skips `#` lines on load ... +#[test] +fn comments_flag_skips_hash_lines() -> TestResult { + assert_sqls( + &[("hx", "a,b\n#x,1\n2,3\n")], + &["--comments"], + &["SELECT * FROM hx"], + "a,b\n2,3", + ) +} + +/// ... and writeback keeps them in place. +#[test] +fn comments_flag_writeback_keeps_hash_lines() -> TestResult { + assert_write_back( + "hx", + b"a,b\n#x,1\n2,3\n", + &["--comments"], + &["UPDATE hx SET b = 9 WHERE a = 2"], + b"a,b\n#x,1\n2,9\n", + ) +} + +// =========================================================================== +// C. CLI / REPL / errors +// =========================================================================== + +// --------------------------------------------------------------------------- +// C15. CLI flags (cli.rs, config.rs, main.rs) +// --------------------------------------------------------------------------- + +/// `-f FILE` runs SQL from a file. +#[test] +fn sql_from_file() -> TestResult { + let dir = create_temp_dir()?; + write_files( + dir.path(), + &[("one.csv", b"id\n1\n"), ("q.sql", b"SELECT 1 AS v;\n")], + )?; + let out = run_in(dir.path(), &["-f", "q.sql", "one.csv"], None)?; + assert!( + out.status.success(), + "{}", + String::from_utf8_lossy(&out.stderr) + ); + assert_eq!(String::from_utf8_lossy(&out.stdout), "v\n1\n"); + Ok(()) +} + +/// `-f -` reads SQL from stdin. +#[test] +fn sql_from_stdin() -> TestResult { + let dir = create_temp_dir()?; + write_files(dir.path(), &[("one.csv", b"id\n1\n")])?; + let out = run_in(dir.path(), &["-f", "-", "one.csv"], Some(b"SELECT 2 AS v"))?; + assert!( + out.status.success(), + "{}", + String::from_utf8_lossy(&out.stderr) + ); + assert_eq!(String::from_utf8_lossy(&out.stdout), "v\n2\n"); + Ok(()) +} + +/// `-s` and `-f` run in command-line order. +#[test] +fn sql_and_files_run_in_order() -> TestResult { + let dir = create_temp_dir()?; + write_files( + dir.path(), + &[("one.csv", b"id\n1\n"), ("q2.sql", b"SELECT 2 AS v;\n")], + )?; + let out = run_in( + dir.path(), + &[ + "-s", + "SELECT 1 AS v", + "-f", + "q2.sql", + "-s", + "SELECT 3 AS v", + "one.csv", + ], + None, + )?; + assert!( + out.status.success(), + "{}", + String::from_utf8_lossy(&out.stderr) + ); + assert_eq!(String::from_utf8_lossy(&out.stdout), "v\n1\nv\n2\nv\n3\n"); + Ok(()) +} + +/// `--output-delimiter` sets the output separator (and overrides -F's). +#[test] +fn output_delimiter_flag() -> TestResult { + assert_sqls( + &[("p.txt", "a|b\n1|2\n")], + &["-F", "|", "--output-delimiter", ";"], + &["SELECT * FROM p"], + "a;b\n1;2", + ) +} + +/// `--no-output-header` omits the header line. +#[test] +fn no_output_header_flag() -> TestResult { + assert_sqls( + &[("rw.csv", "a,b\n1,2\n")], + &["--no-output-header"], + &["SELECT * FROM rw"], + "1,2", + ) +} + +/// `--field-separator` is the long form of -F. +#[test] +fn field_separator_long_flag() -> TestResult { + assert_sqls( + &[("c.txt", "a:b\n1:2\n")], + &["--field-separator", ":"], + &["SELECT b FROM c"], + "b\n2", + ) +} + +/// `-i` with `-s` silently ignores `-s` and starts the REPL. It is a usage +/// error (exit 2). +#[test] +fn interactive_with_sql_is_usage_error() -> TestResult { + let dir = create_temp_dir()?; + write_files(dir.path(), &[("one.csv", b"id\n1\n")])?; + let out = run_in(dir.path(), &["-i", "-s", "SELECT 1", "one.csv"], Some(b""))?; + assert_eq!(out.status.code(), Some(2)); + Ok(()) +} + +// --------------------------------------------------------------------------- +// C16. Error format: one line `sqawk: `, exit 1 (error.rs ~71-103 +// prints Display and then the source chain, repeating the cause) +// --------------------------------------------------------------------------- + +/// A directory as input prints `Error: ... Caused by: 0: I/O error: Is a +/// directory ... 1: Is a directory ...`. +#[test] +fn error_directory_input() -> TestResult { + let dir = create_temp_dir()?; + fs::create_dir(dir.path().join("adir.csv"))?; + let out = run_in(dir.path(), &["-s", "SELECT 1", "adir.csv"], None)?; + assert_clean_error(&out, &["adir.csv", "Is a directory"]); + let stderr = String::from_utf8_lossy(&out.stderr); + assert_eq!(stderr.matches("Is a directory").count(), 1, "{stderr}"); + Ok(()) +} + +/// A syntax error prints a multi-line chain repeating the parser message. +#[test] +fn error_syntax() -> TestResult { + let out = run_one("SELEC 1")?; + assert_clean_error(&out, &["SELEC"]); + Ok(()) +} + +/// An unknown column prints `Error: Failed to execute SQL: ` plus a +/// "Caused by" line. +#[test] +fn error_unknown_column() -> TestResult { + let out = run_one("SELECT nope FROM one")?; + assert_clean_error(&out, &["nope"]); + Ok(()) +} + +/// An unknown function reports "Unsupported function: FOO"; SQLite's message +/// is `no such function: FOO`. +#[test] +fn error_unknown_function() -> TestResult { + let out = run_one("SELECT foo(id) FROM one")?; + assert_clean_error(&out, &["no such function: FOO"]); + Ok(()) +} + +/// An unreadable input file reports "Permission denied" twice and does not +/// name the file in the cause. +#[cfg(unix)] +#[test] +fn error_permission_denied_names_file() -> TestResult { + let dir = create_temp_dir()?; + if !perms_enforced(dir.path()) { + return Ok(()); + } + let p = write_files(dir.path(), &[("np.csv", b"a\n1\n")])?; + chmod(Path::new(&p[0]), 0o000); + let out = run_in(dir.path(), &["-s", "SELECT 1", "np.csv"], None)?; + chmod(Path::new(&p[0]), 0o644); + assert_clean_error(&out, &["np.csv", "Permission denied"]); + Ok(()) +} + +/// A write failure (read-only directory) names the internal temp file +/// (`.tmpXXXX`). +#[cfg(unix)] +#[test] +fn error_write_failure_hides_temp_file() -> TestResult { + let dir = create_temp_dir()?; + if !perms_enforced(dir.path()) { + return Ok(()); + } + write_files(dir.path(), &[("rod/e.csv", b"a\n1\n")])?; + let rod = dir.path().join("rod"); + chmod(&rod, 0o555); + let out = run_in( + dir.path(), + &["-w", "-s", "DELETE FROM e", "rod/e.csv"], + None, + ); + chmod(&rod, 0o755); + let out = out?; + assert_clean_error(&out, &["e.csv"]); + // The test's own directory is a `.tmpXXXX` too; look past it. + let stderr = String::from_utf8_lossy(&out.stderr); + let base = fs::canonicalize(dir.path())?; + let rest = stderr + .replace(&base.display().to_string(), "") + .replace(&dir.path().display().to_string(), ""); + assert!(!rest.contains(".tmp"), "temp file name leaked: {stderr}"); + Ok(()) +} + +// --------------------------------------------------------------------------- +// C17. `-v` panics when stderr is closed (`eprintln!`) +// --------------------------------------------------------------------------- + +/// With stderr's read end closed, `-v` exits 101 ("failed printing to stderr"). +#[test] +fn verbose_with_closed_stderr_does_not_panic() -> TestResult { + let dir = create_temp_dir()?; + // 20,000 rows make -v write megabytes to stderr, far beyond a pipe + // buffer, so a write after the read end is closed is certain. + let mut big = String::from("n\n"); + for i in 0..20_000 { + big.push_str(&format!("{i}\n")); + } + write_files(dir.path(), &[("big.csv", big.as_bytes())])?; + let mut child = Command::new(sqawk_bin()) + .args(["-v", "-s", "SELECT n FROM big", "big.csv"]) + .current_dir(dir.path()) + .stdout(Stdio::null()) + .stderr(Stdio::piped()) + .spawn()?; + let mut first = [0u8; 16]; + child.stderr.as_mut().unwrap().read_exact(&mut first)?; + drop(child.stderr.take()); + let status = child.wait()?; + assert_ne!(status.code(), Some(101), "panicked on closed stderr"); + assert!(status.code().is_some(), "killed by signal: {status:?}"); + Ok(()) +} + +// --------------------------------------------------------------------------- +// C18. Piped (non-tty) REPL (repl.rs) +// --------------------------------------------------------------------------- + +/// Piped REPL stdout carries the banner, "Query returned 1 rows" and +/// "CTRL-D" around the result. Only query results belong on stdout. +#[test] +fn repl_piped_stdout_is_results_only() -> TestResult { + let (out, _d) = run_repl(&[("rw.csv", "a,b\n1,2\n")], "SELECT a FROM rw;\n")?; + assert!(out.status.success()); + assert_eq!(String::from_utf8_lossy(&out.stdout), "a\n1\n"); + Ok(()) +} + +/// `.exit 3` exits with status 0. +#[test] +fn repl_exit_code() -> TestResult { + let (out, _d) = run_repl(&[("rw.csv", "a,b\n1,2\n")], ".exit 3\n")?; + assert_eq!(out.status.code(), Some(3)); + Ok(()) +} + +/// A SQL error in a piped script still exits 0. +#[test] +fn repl_script_error_exits_nonzero() -> TestResult { + let (out, _d) = run_repl(&[("rw.csv", "a,b\n1,2\n")], "SELECT nope FROM rw;\n")?; + assert!(!out.status.success(), "exit 0 after SQL error"); + Ok(()) +} + +/// A final statement without `;` at EOF is silently dropped. +#[test] +fn repl_unterminated_last_statement_runs() -> TestResult { + let (out, _d) = run_repl(&[("rw.csv", "a,b\n1,2\n")], "SELECT 2 AS v")?; + assert!( + String::from_utf8_lossy(&out.stdout).contains("v\n2\n"), + "stdout:\n{}", + String::from_utf8_lossy(&out.stdout) + ); + Ok(()) +} + +/// A piped REPL writes `.sqawk_history` into the current directory. History +/// is kept only for a tty (in `$XDG_STATE_HOME/sqawk/history` or +/// `~/.sqawk_history`). +#[test] +fn repl_piped_writes_no_history() -> TestResult { + let (_out, dir) = run_repl(&[("rw.csv", "a,b\n1,2\n")], "SELECT a FROM rw;\n")?; + fn walk(p: &Path, found: &mut Vec) { + for e in fs::read_dir(p).unwrap().flatten() { + let path = e.path(); + if path.is_dir() { + walk(&path, found); + } else if path.to_string_lossy().contains("hist") { + found.push(path); + } + } + } + let mut found = Vec::new(); + walk(dir.path(), &mut found); + assert!(found.is_empty(), "history written: {found:?}"); + Ok(()) +} + +// --------------------------------------------------------------------------- +// C19. REPL `.tables PATTERN` uses a regex, not LIKE (repl.rs) +// --------------------------------------------------------------------------- + +/// `.tables e%` misses `Ebig`: LIKE is ASCII case-insensitive. +#[test] +fn repl_tables_pattern_case_insensitive() -> TestResult { + assert_eq!(repl_tables("e%")?, vec!["Ebig", "emp", "exa"]); + Ok(()) +} + +/// `.tables e.a` lists `exa`: `.` is a literal in LIKE. +#[test] +fn repl_tables_pattern_dot_is_literal() -> TestResult { + assert_eq!(repl_tables("e.a")?, Vec::::new()); + Ok(()) +} + +// --------------------------------------------------------------------------- +// C20. Scalar functions and operators (func.rs, datetime.rs, arith.rs) +// --------------------------------------------------------------------------- + +/// `printf('%.3d', 7)` ignores the precision (prints `7`). +#[test] +fn printf_integer_precision_pads() -> TestResult { + one("SELECT printf('%.3d', 7) AS v FROM one", "v\n007") +} + +/// `printf('%u', -1)` prints `-1`; SQLite reinterprets as unsigned 64-bit. +#[test] +fn printf_unsigned() -> TestResult { + one( + "SELECT printf('%u', -1) AS v FROM one", + "v\n18446744073709551615", + ) +} + +/// A negative year is rejected (NULL). +#[test] +fn date_negative_year() -> TestResult { + one("SELECT date('-0001-12-31') AS v FROM one", "v\n-0001-12-31") +} + +/// A trailing space after the date is rejected (NULL). +#[test] +fn date_trailing_space() -> TestResult { + one("SELECT date('2024-01-01 ') AS v FROM one", "v\n2024-01-01") +} + +/// `strftime('%u', ...)` (ISO weekday, Sunday = 7) yields NULL. +#[test] +fn strftime_iso_weekday() -> TestResult { + one("SELECT strftime('%u', '2024-01-07') AS v FROM one", "v\n7") +} + +/// The `subsec` modifier yields NULL. +#[test] +fn date_modifier_subsec() -> TestResult { + one( + "SELECT datetime('2024-01-01 10:00:00.123', 'subsec') AS v FROM one", + "v\n2024-01-01 10:00:00.123", + ) +} + +/// The `auto` modifier yields NULL. +#[test] +fn date_modifier_auto() -> TestResult { + one( + "SELECT datetime(1700000000, 'auto') AS v FROM one", + "v\n2023-11-14 22:13:20", + ) +} + +/// The `julianday` modifier yields NULL. +#[test] +fn date_modifier_julianday() -> TestResult { + one( + "SELECT date(2460000.5, 'julianday') AS v FROM one", + "v\n2023-02-25", + ) +} + +/// The `+HH:MM` modifier yields NULL. +#[test] +fn date_modifier_time_offset() -> TestResult { + one( + "SELECT datetime('2024-01-01 10:00:00', '+01:30') AS v FROM one", + "v\n2024-01-01 11:30:00", + ) +} + +/// The `+YYYY-MM-DD` modifier yields NULL. +#[test] +fn date_modifier_date_offset() -> TestResult { + one( + "SELECT date('2024-01-01', '+0001-02-03') AS v FROM one", + "v\n2025-03-04", + ) +} + +/// `0x10` fails with "Literal X'10' is not supported" (the parser reads it as +/// a blob literal). +#[test] +fn hex_integer_literal() -> TestResult { + one("SELECT 0x10 AS v FROM one", "v\n16") +} + +/// Bitwise operators fail to parse (`<<`) or are unsupported. +#[test] +fn bitwise_operators() -> TestResult { + one( + "SELECT 5 & 3 AS a, 5 | 3 AS b, ~5 AS c, 1 << 4 AS d, 256 >> 4 AS e FROM one", + "a,b,c,d,e\n1,7,-6,16,16", + ) +} + +/// Bitwise operators bind below `+` and above `=` in SQLite: +/// `(1 + 2) << 1` = 6 and `(6 & 3) = 2` = 1. +#[test] +fn bitwise_operator_precedence() -> TestResult { + one( + "SELECT 1 + 2 << 1 AS a, 6 & 3 = 2 AS b FROM one", + "a,b\n6,1", + ) +} + +// =========================================================================== +// D. Transactions: BEGIN / COMMIT / ROLLBACK over the in-memory tables +// =========================================================================== + +/// BEGIN; UPDATE; ROLLBACK restores the original values. +#[test] +fn rollback_restores_values() -> TestResult { + assert_sqls( + &[("tx.csv", TX)], + &[], + &[ + "BEGIN", + "UPDATE tx SET n = 0", + "ROLLBACK", + "SELECT n FROM tx", + ], + "n\n10\n20", + ) +} + +/// BEGIN; INSERT; COMMIT keeps the insert (statements in one `-s`). +#[test] +fn commit_keeps_changes() -> TestResult { + assert_sqls( + &[("tx.csv", TX)], + &[], + &["BEGIN; INSERT INTO tx VALUES (3, 30); COMMIT; SELECT k FROM tx"], + "k\n1\n2\n3", + ) +} + +/// A committed change is written back. +#[test] +fn commit_is_written_back() -> TestResult { + assert_write_back( + "tx.csv", + TX.as_bytes(), + &[], + &["BEGIN", "INSERT INTO tx VALUES (3, 30)", "COMMIT"], + b"k,n\n1,10\n2,20\n3,30\n", + ) +} + +/// After ROLLBACK, --write leaves the file byte-identical. +#[test] +fn rollback_is_not_written_back() -> TestResult { + let content = b"k,n\n1,\"10\"\n2,20\n"; + assert_write_back( + "tx.csv", + content, + &[], + &["BEGIN", "UPDATE tx SET n = 0", "ROLLBACK"], + content, + ) +} + +/// A transaction still open at exit is rolled back before --write. +#[test] +fn open_transaction_at_exit_is_rolled_back() -> TestResult { + assert_write_back( + "tx.csv", + TX.as_bytes(), + &[], + &["BEGIN", "UPDATE tx SET n = 0"], + TX.as_bytes(), + ) +} + +/// BEGIN inside BEGIN is an error (SQLite: "cannot start a transaction +/// within a transaction"). +#[test] +fn nested_begin_is_error() -> TestResult { + let out = run_sqls(&[("tx.csv", TX)], &[], &["BEGIN", "BEGIN"])?; + assert_clean_error(&out, &["within a transaction"]); + Ok(()) +} + +/// COMMIT without BEGIN is an error (SQLite: "cannot commit - no transaction +/// is active"). +#[test] +fn commit_without_begin_is_error() -> TestResult { + let out = run_sqls(&[("tx.csv", TX)], &[], &["COMMIT"])?; + assert_clean_error(&out, &["no transaction is active"]); + Ok(()) +} + +/// ROLLBACK without BEGIN is an error (SQLite: "cannot rollback - no +/// transaction is active"). +#[test] +fn rollback_without_begin_is_error() -> TestResult { + let out = run_sqls(&[("tx.csv", TX)], &[], &["ROLLBACK"])?; + assert_clean_error(&out, &["no transaction is active"]); + Ok(()) +} + +/// SAVEPOINT, RELEASE and ROLLBACK TO are rejected with a clean +/// "not supported" error. +#[test] +fn savepoints_are_not_supported() -> TestResult { + for stmt in ["SAVEPOINT s", "RELEASE s", "ROLLBACK TO s"] { + let out = run_sqls(&[("tx.csv", TX)], &[], &["BEGIN", stmt])?; + assert_clean_error(&out, &["not supported"]); + } + Ok(()) +} + +/// A piped REPL script: BEGIN; UPDATE; ROLLBACK; SELECT shows the original +/// values. +#[test] +fn repl_rollback_restores_values() -> TestResult { + let (out, _d) = run_repl( + &[("tx.csv", TX)], + "BEGIN;\nUPDATE tx SET n = 0;\nROLLBACK;\nSELECT n FROM tx;\n", + )?; + let stdout = String::from_utf8_lossy(&out.stdout); + assert!(stdout.contains("n\n10\n20\n"), "stdout:\n{stdout}"); + assert!( + out.status.success(), + "{}", + String::from_utf8_lossy(&out.stderr) + ); + Ok(()) +} + +// --------------------------------------------------------------------------- +// ORDER BY / GROUP BY integer terms (found after P9) +// --------------------------------------------------------------------------- +// +// SQLite's rule: an integer literal, optionally under parentheses and unary +// `+` or `-`, whose +// value fits in 32 bits is a result-column position and must lie in 1..=N or +// the statement fails with "Nth ORDER BY term out of range"; any other term +// (a larger integer, a real such as 1.0, an expression such as 1+0) is an +// ordinary expression, so a constant term does not reorder rows. + +const T3: (&str, &str) = ("t3.csv", "a\n3\n1\n2\n"); + +/// `ORDER BY +1` is position 1 in SQLite and sorts the rows; sqawk treated it +/// as a constant and returned them unsorted -- a silent wrong answer. +#[test] +fn order_by_unary_plus_position_sorts() -> TestResult { + assert_query_on(&[T3], "SELECT a FROM t3 ORDER BY +1", "a\n1\n2\n3")?; + assert_query_on(&[T3], "SELECT a FROM t3 ORDER BY +1 DESC", "a\n3\n2\n1") +} + +/// `GROUP BY +1` is position 1 too. +#[test] +fn group_by_unary_plus_position() -> TestResult { + assert_query_on( + &[T3], + "SELECT a, COUNT(*) AS n FROM t3 GROUP BY +1 ORDER BY 1", + "a,n\n1,1\n2,1\n3,1", + ) +} + +/// Negative and zero positions are out of range (sqawk accepted `-1` as a +/// constant and `GROUP BY -1` gave a misleading GROUP BY membership error). +#[test] +fn negative_and_zero_positions_are_out_of_range() -> TestResult { + for sql in [ + "SELECT a FROM t3 ORDER BY -1", + "SELECT a FROM t3 ORDER BY -0", + "SELECT a FROM t3 ORDER BY 0", + "SELECT a FROM t3 ORDER BY 2147483647", + ] { + assert_query_on_fails(&[T3], sql, "ORDER BY term out of range")?; + } + for sql in [ + "SELECT a FROM t3 GROUP BY 0", + "SELECT a FROM t3 GROUP BY -1", + ] { + assert_query_on_fails(&[T3], sql, "GROUP BY term out of range")?; + } + Ok(()) +} + +/// Integers beyond 32 bits and real literals are constant expressions, not +/// positions: no reordering, a single group (sqawk errored on both). +#[test] +fn large_integer_and_real_terms_are_constants() -> TestResult { + assert_query_on(&[T3], "SELECT a FROM t3 ORDER BY 4294967297", "a\n3\n1\n2")?; + assert_query_on(&[T3], "SELECT a FROM t3 ORDER BY 2147483648", "a\n3\n1\n2")?; + assert_query_on(&[T3], "SELECT a FROM t3 ORDER BY 1.0", "a\n3\n1\n2")?; + assert_query_on( + &[T3], + "SELECT COUNT(*) AS n FROM t3 GROUP BY 4294967297", + "n\n3", + ) +} diff --git a/tests/distinct/mod.rs b/tests/distinct/mod.rs index 4c4c4c9..bd7674d 100644 --- a/tests/distinct/mod.rs +++ b/tests/distinct/mod.rs @@ -199,10 +199,7 @@ fn test_distinct_with_join() -> Result<(), Box> { ); // Check output - should include unique combinations from the join - assert!( - stdout.contains("users.name,orders.product_id"), - "Missing header" - ); + assert!(stdout.contains("name,product_id"), "Missing header"); // Check for expected entries after join let expected_entries = vec!["John,101", "John,103", "John,104", "Jane,102", "Jane,105"]; diff --git a/tests/exists/mod.rs b/tests/exists/mod.rs index b1f8b5f..ecbb176 100644 --- a/tests/exists/mod.rs +++ b/tests/exists/mod.rs @@ -1,7 +1,6 @@ //! EXISTS/NOT EXISTS subquery tests for sqawk VM //! //! This file contains tests for EXISTS subqueries. -//! Phase 4F: Subquery support use crate::helpers::{create_custom_csv, create_temp_dir}; diff --git a/tests/file_io/mod.rs b/tests/file_io/mod.rs new file mode 100644 index 0000000..c49bb3a --- /dev/null +++ b/tests/file_io/mod.rs @@ -0,0 +1,310 @@ +//! File IO: file-type rules, table specs, and byte-exact writeback. +//! +//! Every assertion is exact: complete stdout, or the complete file bytes +//! after `--write`. + +use std::fs; +use std::path::Path; +use std::process::{Command, Stdio}; + +use crate::helpers::{assert_query_on, create_temp_dir, run_sql}; + +type TestResult = Result<(), Box>; + +/// Write `content` to `dir/name`, returning the path as a string. +fn put(dir: &Path, name: &str, content: &[u8]) -> Result> { + let p = dir.join(name); + fs::write(&p, content)?; + Ok(p.to_str().unwrap().to_string()) +} + +/// Run `sql` with `--write` (plus `extra`) over a temp file `name` holding +/// `content`; require success and return the file's bytes afterwards. +fn write_back( + name: &str, + content: &[u8], + extra: &[&str], + sql: &str, +) -> Result, Box> { + let dir = create_temp_dir()?; + let p = put(dir.path(), name, content)?; + let mut args = extra.to_vec(); + args.push("--write"); + let (_out, stderr, ok) = run_sql(sql, &[&p], &args)?; + assert!(ok, "write failed\n sql: {sql}\n stderr: {stderr}"); + Ok(fs::read(&p)?) +} + +fn assert_write_back( + name: &str, + content: &[u8], + extra: &[&str], + sql: &str, + expected: &[u8], +) -> TestResult { + let actual = write_back(name, content, extra, sql)?; + assert_eq!( + String::from_utf8_lossy(&actual), + String::from_utf8_lossy(expected), + "\n=== sql ===\n{sql}\n" + ); + assert_eq!(actual, expected); + Ok(()) +} + +// --------------------------------------------------------------------------- +// File-type rules +// --------------------------------------------------------------------------- + +#[test] +fn tab_extension_is_tab_separated() -> TestResult { + assert_query_on(&[("t.tab", "a\tb\n1\t2\n")], "SELECT b FROM t", "b\n2")?; + assert_query_on(&[("u.TSV", "a\tb\n1\t2\n")], "SELECT b FROM u", "b\n2") +} + +#[test] +fn field_separator_overrides_extension() -> TestResult { + let dir = create_temp_dir()?; + let p = put(dir.path(), "t.tsv", b"a;b\n1;2\n")?; + let (out, stderr, ok) = run_sql("SELECT b FROM t", &[&p], &["-F;"])?; + assert!(ok, "{stderr}"); + assert_eq!(out, "b\n2\n"); + Ok(()) +} + +#[test] +fn stdin_is_comma_separated_with_header() -> TestResult { + let mut child = Command::new(env!("CARGO_BIN_EXE_sqawk")) + .args(["-s", "SELECT b FROM stdin", "-"]) + .stdin(Stdio::piped()) + .stdout(Stdio::piped()) + .spawn()?; + std::io::Write::write_all(&mut child.stdin.take().unwrap(), b"a,b\n1,2\n")?; + let out = child.wait_with_output()?; + assert!(out.status.success()); + assert_eq!(String::from_utf8(out.stdout)?, "b\n2\n"); + Ok(()) +} + +#[test] +fn comments_flag_applies_to_csv_too() -> TestResult { + assert_write_back( + "c.csv", + b"# head\na,b\n# mid\n1,2\n3,4\n# tail\n", + &["--comments"], + "DELETE FROM c WHERE a = 1", + b"# head\na,b\n# mid\n3,4\n# tail\n", + ) +} + +// --------------------------------------------------------------------------- +// Verbatim writeback of untouched rows +// --------------------------------------------------------------------------- + +/// Blank lines are kept where they were. +#[test] +fn blank_lines_are_kept() -> TestResult { + assert_write_back( + "b.csv", + b"a,b\n\n1,2\n\n\n3,4\n\n", + &[], + "UPDATE b SET b = 9 WHERE a = 3", + b"a,b\n\n1,2\n\n\n3,9\n\n", + ) +} + +/// A CRLF file whose last line has no line ending gets CRLF before an +/// appended row; the untouched rows keep their bytes. +#[test] +fn crlf_insert_after_unterminated_last_line() -> TestResult { + assert_write_back( + "c.csv", + b"k,s\r\n1,\"a\"\r\n2,b", + &[], + "INSERT INTO c VALUES (3, 'c')", + b"k,s\r\n1,\"a\"\r\n2,b\r\n3,c\r\n", + ) +} + +/// Mixed line endings in untouched rows survive. +#[test] +fn mixed_line_endings_are_kept() -> TestResult { + let content = b"k,s\r\n1,a\n2,b\r\n3,c\n"; + assert_write_back("m.csv", content, &[], "UPDATE m SET k = k", content) +} + +/// An UPDATE that changes a row writes that row from its values; the other +/// rows keep their bytes. +#[test] +fn changed_row_is_rewritten() -> TestResult { + assert_write_back( + "q.csv", + b"k,s\n1,\"abc\"\n2,\"abc\"\n", + &[], + "UPDATE q SET s = 'x' WHERE k = 2", + b"k,s\n1,\"abc\"\n2,x\n", + ) +} + +/// A value replaced by an equal value of another type (5 -> 5.0) is a change. +#[test] +fn update_to_different_type_is_a_change() -> TestResult { + assert_write_back( + "q.csv", + b"k,s\n1,5\n2,5\n", + &[], + "UPDATE q SET s = 5.0 WHERE k = 2", + b"k,s\n1,5\n2,5.0\n", + ) +} + +/// ALTER TABLE ADD COLUMN changes every line, header included. +#[test] +fn alter_add_column_rewrites_every_line() -> TestResult { + assert_write_back( + "q.csv", + b"\"k\",s\n1,\"abc\"\n", + &[], + "ALTER TABLE q ADD COLUMN t TEXT", + b"k,s,t\n1,abc,\n", + ) +} + +/// DELETE keeps the surviving rows' bytes; the BOM stays. +#[test] +fn delete_keeps_other_rows_and_bom() -> TestResult { + assert_write_back( + "d.csv", + b"\xef\xbb\xbfk,s\n1, a \n2,b\n3,\"c\"", + &[], + "DELETE FROM d WHERE k = 2", + b"\xef\xbb\xbfk,s\n1, a \n3,\"c\"", + ) +} + +// --------------------------------------------------------------------------- +// Malformed input +// --------------------------------------------------------------------------- + +/// An unclosed quote is a load error naming the line where the field +/// starts; with --write nothing is written. +#[test] +fn unclosed_quote_is_load_error() -> TestResult { + let dir = create_temp_dir()?; + let content = b"a,b\n1,2\n3,\"x\n4,5\n"; + let p = put(dir.path(), "u.csv", content)?; + let (_out, stderr, ok) = run_sql("DELETE FROM u", &[&p], &["--write"])?; + assert!(!ok); + assert!( + stderr.contains("u.csv: line 3: unterminated quoted field"), + "{stderr}" + ); + assert_eq!(fs::read(&p)?, content); + Ok(()) +} + +/// Load errors name the same line in a CRLF file as in its LF twin: too +/// many fields, an unclosed quote, and both past `#` lines under --comments. +#[test] +fn load_error_lines_are_the_same_for_crlf() -> TestResult { + let dir = create_temp_dir()?; + let cases: [(&str, &str, &[&str], &str); 4] = [ + ( + "w", + "id,name\n1,\"a\nb\"\n\n2,x\n3,y,z\n", + &[], + "line 6: record has 3 fields, but the table has 2 columns", + ), + ( + "q", + "a,b\n1,2\n3,\"x\n4,5\n", + &[], + "line 3: unterminated quoted field", + ), + ( + "cw", + "# head\na,b\n# mid\n1,2\n3,4,5\n", + &["--comments"], + "line 5: record has 3 fields", + ), + ( + "cq", + "# head\na,b\n# mid\n1,\"x\n", + &["--comments"], + "line 4: unterminated quoted field", + ), + ]; + for (name, lf, flags, want) in cases { + for (suffix, content) in [("lf", lf.to_string()), ("crlf", lf.replace('\n', "\r\n"))] { + let file = format!("{name}_{suffix}.csv"); + let p = put(dir.path(), &file, content.as_bytes())?; + let (_out, stderr, ok) = run_sql("SELECT 1", &[&p], flags)?; + assert!(!ok, "{file}"); + assert!(stderr.contains(&format!("{file}: {want}")), "{stderr}"); + } + } + Ok(()) +} + +#[test] +fn duplicate_header_names_are_named() -> TestResult { + let dir = create_temp_dir()?; + let p = put(dir.path(), "h.csv", b"id,Name,x,NAME,y\n1,2,3,4,5\n")?; + let (_out, stderr, ok) = run_sql("SELECT 1", &[&p], &[])?; + assert!(!ok); + assert!(stderr.contains("duplicate column name: NAME"), "{stderr}"); + Ok(()) +} + +// --------------------------------------------------------------------------- +// Column names are case-insensitive in every statement +// --------------------------------------------------------------------------- + +#[test] +fn insert_and_update_columns_are_case_insensitive() -> TestResult { + assert_write_back( + "c.csv", + b"Id,Name\n1,a\n", + &[], + "INSERT INTO c (ID, NAME) VALUES (2, 'b'); UPDATE c SET name = 'z' WHERE id = 1", + b"Id,Name\n1,z\n2,b\n", + ) +} + +// --------------------------------------------------------------------------- +// Table specs +// --------------------------------------------------------------------------- + +#[test] +fn name_prefix_names_the_table() -> TestResult { + let dir = create_temp_dir()?; + put(dir.path(), "data.csv", b"a\n1\n")?; + let spec = format!("t_1={}", dir.path().join("data.csv").display()); + let (out, stderr, ok) = run_sql("SELECT a FROM t_1", &[&spec], &[])?; + assert!(ok, "{stderr}"); + assert_eq!(out, "a\n1\n"); + Ok(()) +} + +#[test] +fn file_name_without_stem_is_error() -> TestResult { + let dir = create_temp_dir()?; + let p = put(dir.path(), ".csv", b"a\n1\n")?; + let (_out, stderr, ok) = run_sql("SELECT 1", &[&p], &[])?; + assert!(!ok); + assert!(stderr.contains("cannot derive a table name"), "{stderr}"); + Ok(()) +} + +#[cfg(unix)] +#[test] +fn same_file_through_symlink_is_error() -> TestResult { + let dir = create_temp_dir()?; + let p = put(dir.path(), "a.csv", b"a\n1\n")?; + let link = dir.path().join("b.csv"); + std::os::unix::fs::symlink(&p, &link)?; + let (_out, stderr, ok) = run_sql("SELECT 1", &[&p, link.to_str().unwrap()], &[])?; + assert!(!ok); + assert!(stderr.contains("is already loaded as table a"), "{stderr}"); + Ok(()) +} diff --git a/tests/golden/mod.rs b/tests/golden/mod.rs index 4fd6c89..cb73b2d 100644 --- a/tests/golden/mod.rs +++ b/tests/golden/mod.rs @@ -11,7 +11,7 @@ //! //! Known-wrong behaviour is NOT pinned here -- it lives in `tests/defects/`. -use crate::helpers::assert_query; +use crate::helpers::{assert_query, assert_query_on, assert_query_on_fails, ONE}; #[test] fn select_star() -> Result<(), Box> { @@ -56,7 +56,7 @@ fn select_literal() -> Result<(), Box> { assert_query( &["tests/data/employees.csv"], "SELECT 1", - concat!("expr", "\n1",), + concat!("1", "\n1",), ) } @@ -67,7 +67,7 @@ fn select_literal_expressions() -> Result<(), Box> { assert_query( &["tests/data/employees.csv"], "SELECT UPPER('ab') || 'c', CAST('12' AS INTEGER) + 1, CASE WHEN 1 > 2 THEN 'x' ELSE 'y' END, SUBSTR('hello', 2, 3), -ABS(-4)", - concat!("expr,expr,expr,SUBSTR(expr),expr", "\nABc,13,y,ell,-4"), + concat!("UPPER('ab') || 'c',CAST('12' AS INTEGER) + 1,CASE WHEN 1 > 2 THEN 'x' ELSE 'y' END,\"SUBSTR('hello', 2, 3)\",-ABS(-4)", "\nABc,13,y,ell,-4"), ) } @@ -77,7 +77,7 @@ fn select_three_valued_predicates() -> Result<(), Box> { assert_query( &["tests/data/employees.csv"], "SELECT NULL AND 0, NULL OR 1, NULL OR 0, 2 IN (1, NULL), 1 IN (1, NULL), NULL BETWEEN 1 AND 2, NULL LIKE 'a%'", - concat!("expr,expr,expr,expr,expr,expr,expr", "\n0,1,,,1,,"), + concat!("NULL AND 0,NULL OR 1,NULL OR 0,\"2 IN (1, NULL)\",\"1 IN (1, NULL)\",NULL BETWEEN 1 AND 2,NULL LIKE 'a%'", "\n0,1,,,1,,"), ) } @@ -342,7 +342,7 @@ fn agg_count() -> Result<(), Box> { assert_query( &["tests/data/employees.csv"], "SELECT COUNT(*) FROM employees", - concat!("COUNT", "\n8",), + concat!("COUNT(*)", "\n8",), ) } @@ -351,7 +351,7 @@ fn agg_sum() -> Result<(), Box> { assert_query( &["tests/data/employees.csv"], "SELECT SUM(salary) FROM employees", - concat!("SUM", "\n540000",), + concat!("SUM(salary)", "\n540000",), ) } @@ -360,7 +360,7 @@ fn agg_avg() -> Result<(), Box> { assert_query( &["tests/data/employees.csv"], "SELECT AVG(salary) FROM employees", - concat!("AVG", "\n67500",), + concat!("AVG(salary)", "\n67500.0",), ) } @@ -369,7 +369,7 @@ fn agg_min_max() -> Result<(), Box> { assert_query( &["tests/data/employees.csv"], "SELECT MIN(salary), MAX(salary) FROM employees", - concat!("MIN,MAX", "\n45000,90000",), + concat!("MIN(salary),MAX(salary)", "\n45000,90000",), ) } @@ -378,7 +378,7 @@ fn agg_with_where() -> Result<(), Box> { assert_query( &["tests/data/employees.csv"], "SELECT COUNT(*) FROM employees WHERE department = 'Engineering'", - concat!("COUNT", "\n3",), + concat!("COUNT(*)", "\n3",), ) } @@ -397,7 +397,7 @@ fn group_by_basic() -> Result<(), Box> { &["tests/data/employees.csv"], "SELECT department, COUNT(*) FROM employees GROUP BY department", concat!( - "department,COUNT", + "department,COUNT(*)", "\nEngineering,3", "\nHR,1", "\nMarketing,2", @@ -412,7 +412,7 @@ fn group_by_sum() -> Result<(), Box> { &["tests/data/employees.csv"], "SELECT department, SUM(salary) FROM employees GROUP BY department", concat!( - "department,SUM", + "department,SUM(salary)", "\nEngineering,210000", "\nHR,45000", "\nMarketing,115000", @@ -427,7 +427,7 @@ fn group_by_having() -> Result<(), Box> { &["tests/data/employees.csv"], "SELECT department, COUNT(*) FROM employees GROUP BY department HAVING COUNT(*) > 1", concat!( - "department,COUNT", + "department,COUNT(*)", "\nEngineering,3", "\nMarketing,2", "\nSales,2", @@ -441,7 +441,7 @@ fn join_on() -> Result<(), Box> { &["tests/data/users.csv", "tests/data/orders.csv"], "SELECT users.name, orders.id FROM users JOIN orders ON users.id = orders.user_id", concat!( - "users.name,orders.id", + "name,id", "\nJohn,101", "\nJohn,103", "\nJohn,105", @@ -457,7 +457,7 @@ fn join_inner() -> Result<(), Box> { &["tests/data/users.csv", "tests/data/orders.csv"], "SELECT users.name, orders.date FROM users INNER JOIN orders ON users.id = orders.user_id", concat!( - "users.name,orders.date", + "name,date", "\nJohn,2023-01-15", "\nJohn,2023-02-10", "\nJohn,2023-03-05", @@ -473,7 +473,7 @@ fn join_left() -> Result<(), Box> { &["tests/data/users.csv", "tests/data/orders.csv"], "SELECT users.name, orders.id FROM users LEFT JOIN orders ON users.id = orders.user_id", concat!( - "users.name,orders.id", + "name,id", "\nJohn,101", "\nJohn,103", "\nJohn,105", @@ -489,7 +489,7 @@ fn join_right() -> Result<(), Box> { &["tests/data/users.csv", "tests/data/orders.csv"], "SELECT users.name, orders.id FROM users RIGHT JOIN orders ON users.id = orders.user_id", concat!( - "users.name,orders.id", + "name,id", "\nJohn,101", "\nJane,102", "\nJohn,103", @@ -505,7 +505,7 @@ fn join_full() -> Result<(), Box> { &["tests/data/users.csv", "tests/data/orders.csv"], "SELECT users.name, orders.id FROM users FULL JOIN orders ON users.id = orders.user_id", concat!( - "users.name,orders.id", + "name,id", "\nJohn,101", "\nJohn,103", "\nJohn,105", @@ -521,7 +521,7 @@ fn join_cross() -> Result<(), Box> { &["tests/data/users.csv", "tests/data/orders.csv"], "SELECT users.name, orders.id FROM users CROSS JOIN orders", concat!( - "users.name,orders.id", + "name,id", "\nJohn,101", "\nJohn,102", "\nJohn,103", @@ -542,7 +542,7 @@ fn join_implicit() -> Result<(), Box> { &["tests/data/users.csv", "tests/data/orders.csv"], "SELECT users.name, orders.id FROM users, orders WHERE users.id = orders.user_id", concat!( - "users.name,orders.id", + "name,id", "\nJohn,101", "\nJohn,103", "\nJohn,105", @@ -558,7 +558,7 @@ fn join_three() -> Result<(), Box> { &["tests/data/users.csv", "tests/data/orders.csv", "tests/data/products.csv"], "SELECT users.name, products.name FROM users, orders, products WHERE users.id = orders.user_id AND orders.product_id = products.product_id", concat!( - "users.name,products.name", + "name,name", "\nJohn,Laptop", "\nJohn,Headphones", "\nJohn,Keyboard", @@ -715,7 +715,7 @@ fn str_substr() -> Result<(), Box> { &["tests/data/employees.csv"], "SELECT SUBSTR(name, 1, 3) FROM employees", concat!( - "SUBSTR(name)", + "\"SUBSTR(name, 1, 3)\"", "\nAli", "\nBob", "\nCha", @@ -734,7 +734,7 @@ fn str_replace() -> Result<(), Box> { &["tests/data/employees.csv"], "SELECT REPLACE(department, 'Engineering', 'Eng') FROM employees", concat!( - "REPLACE(department)", + "\"REPLACE(department, 'Engineering', 'Eng')\"", "\nEng", "\nMarketing", "\nEng", @@ -753,7 +753,7 @@ fn str_trim() -> Result<(), Box> { &["tests/data/strings.csv"], "SELECT TRIM(padded_text) FROM strings", concat!( - "expr", + "TRIM(padded_text)", "\ntrimme", "\nneeds space", "\nwhitespace", @@ -769,7 +769,7 @@ fn str_concat() -> Result<(), Box> { &["tests/data/employees.csv"], "SELECT CONCAT(name, department) FROM employees", concat!( - "CONCAT(name)", + "\"CONCAT(name, department)\"", "\nAliceEngineering", "\nBobMarketing", "\nCharlieEngineering", @@ -788,7 +788,7 @@ fn str_left_right() -> Result<(), Box> { &["tests/data/employees.csv"], "SELECT LEFT(name, 2), RIGHT(name, 2) FROM employees", concat!( - "LEFT(name),RIGHT(name)", + "\"LEFT(name, 2)\",\"RIGHT(name, 2)\"", "\nAl,ce", "\nBo,ob", "\nCh,ie", @@ -828,14 +828,14 @@ fn math_round() -> Result<(), Box> { "SELECT ROUND(salary) FROM employees", concat!( "ROUND(salary)", - "\n70000", - "\n55000", - "\n65000", - "\n80000", - "\n60000", - "\n75000", - "\n90000", - "\n45000", + "\n70000.0", + "\n55000.0", + "\n65000.0", + "\n80000.0", + "\n60000.0", + "\n75000.0", + "\n90000.0", + "\n45000.0", ), ) } @@ -846,7 +846,7 @@ fn arith_add() -> Result<(), Box> { &["tests/data/employees.csv"], "SELECT name, salary + 1000 FROM employees", concat!( - "name,expr", + "name,salary + 1000", "\nAlice,71000", "\nBob,56000", "\nCharlie,66000", @@ -865,7 +865,7 @@ fn arith_mul() -> Result<(), Box> { &["tests/data/employees.csv"], "SELECT name, salary * 2 FROM employees", concat!( - "name,expr", + "name,salary * 2", "\nAlice,140000", "\nBob,110000", "\nCharlie,130000", @@ -884,7 +884,7 @@ fn arith_div() -> Result<(), Box> { &["tests/data/employees.csv"], "SELECT name, salary / 1000 FROM employees", concat!( - "name,expr", + "name,salary / 1000", "\nAlice,70", "\nBob,55", "\nCharlie,65", @@ -903,7 +903,7 @@ fn arith_mod() -> Result<(), Box> { &["tests/data/employees.csv"], "SELECT name, age % 7 FROM employees", concat!( - "name,expr", + "name,age % 7", "\nAlice,2", "\nBob,4", "\nCharlie,0", @@ -922,7 +922,7 @@ fn arith_unary_minus() -> Result<(), Box> { &["tests/data/employees.csv"], "SELECT name, -salary FROM employees", concat!( - "name,expr", + "name,-salary", "\nAlice,-70000", "\nBob,-55000", "\nCharlie,-65000", @@ -941,7 +941,7 @@ fn coalesce() -> Result<(), Box> { &["tests/data/nullable.csv"], "SELECT id, COALESCE(score, 0) FROM nullable", concat!( - "id,COALESCE(score)", + "id,\"COALESCE(score, 0)\"", "\n1,90", "\n2,0", "\n3,70", @@ -957,7 +957,7 @@ fn nullif() -> Result<(), Box> { &["tests/data/employees.csv"], "SELECT name, NULLIF(department, 'HR') FROM employees", concat!( - "name,NULLIF(department)", + "name,\"NULLIF(department, 'HR')\"", "\nAlice,Engineering", "\nBob,Marketing", "\nCharlie,Engineering", @@ -976,7 +976,7 @@ fn win_row_number() -> Result<(), Box> { &["tests/data/employees.csv"], "SELECT name, ROW_NUMBER() OVER (ORDER BY salary DESC) FROM employees", concat!( - "name,ROW_NUMBER", + "name,ROW_NUMBER() OVER (ORDER BY salary DESC)", "\nGrace,1", "\nDavid,2", "\nFrank,3", @@ -995,7 +995,7 @@ fn win_rank() -> Result<(), Box> { &["tests/data/employees.csv"], "SELECT name, RANK() OVER (ORDER BY department) FROM employees", concat!( - "name,RANK", + "name,RANK() OVER (ORDER BY department)", "\nAlice,1", "\nCharlie,1", "\nFrank,1", @@ -1014,7 +1014,7 @@ fn win_dense_rank() -> Result<(), Box> { &["tests/data/employees.csv"], "SELECT name, DENSE_RANK() OVER (ORDER BY department) FROM employees", concat!( - "name,DENSE_RANK", + "name,DENSE_RANK() OVER (ORDER BY department)", "\nAlice,1", "\nCharlie,1", "\nFrank,1", @@ -1033,7 +1033,7 @@ fn win_lag() -> Result<(), Box> { &["tests/data/employees.csv"], "SELECT name, LAG(salary) OVER (ORDER BY salary) FROM employees", concat!( - "name,LAG", + "name,LAG(salary) OVER (ORDER BY salary)", "\nHenry,", "\nBob,45000", "\nEve,55000", @@ -1052,7 +1052,7 @@ fn win_lead() -> Result<(), Box> { &["tests/data/employees.csv"], "SELECT name, LEAD(salary) OVER (ORDER BY salary) FROM employees", concat!( - "name,LEAD", + "name,LEAD(salary) OVER (ORDER BY salary)", "\nHenry,55000", "\nBob,60000", "\nEve,65000", @@ -1118,7 +1118,7 @@ fn outer_join_inner_baseline() -> Result<(), Box> { &["tests/data/customers.csv", "tests/data/purchases.csv"], "SELECT customers.name, purchases.item FROM customers INNER JOIN purchases ON customers.id = purchases.customer_id", concat!( - "customers.name,purchases.item", + "name,item", "\nAnn,Book", "\nAnn,Pen", "\nBen,Desk", @@ -1132,7 +1132,7 @@ fn outer_join_left_nullfill() -> Result<(), Box> { &["tests/data/customers.csv", "tests/data/purchases.csv"], "SELECT customers.name, purchases.item FROM customers LEFT JOIN purchases ON customers.id = purchases.customer_id", concat!( - "customers.name,purchases.item", + "name,item", "\nAnn,Book", "\nAnn,Pen", "\nBen,Desk", @@ -1147,7 +1147,7 @@ fn outer_join_right_nullfill() -> Result<(), Box> { &["tests/data/customers.csv", "tests/data/purchases.csv"], "SELECT customers.name, purchases.item FROM customers RIGHT JOIN purchases ON customers.id = purchases.customer_id", concat!( - "customers.name,purchases.item", + "name,item", "\nAnn,Book", "\nAnn,Pen", "\nBen,Desk", @@ -1162,7 +1162,7 @@ fn outer_join_full_nullfill() -> Result<(), Box> { &["tests/data/customers.csv", "tests/data/purchases.csv"], "SELECT customers.name, purchases.item FROM customers FULL JOIN purchases ON customers.id = purchases.customer_id", concat!( - "customers.name,purchases.item", + "name,item", "\nAnn,Book", "\nAnn,Pen", "\nBen,Desk", @@ -1189,7 +1189,7 @@ fn err_unknown_column() -> Result<(), Box> { assert_query_fails( &["tests/data/employees.csv"], "SELECT nosuchcolumn FROM employees", - "not found", + "no such column: nosuchcolumn", ) } @@ -1198,7 +1198,7 @@ fn err_unknown_table() -> Result<(), Box> { assert_query_fails( &["tests/data/employees.csv"], "SELECT name FROM nosuchtable", - "not found", + "no such table: nosuchtable", ) } @@ -1207,7 +1207,7 @@ fn err_syntax() -> Result<(), Box> { assert_query_fails( &["tests/data/employees.csv"], "SELECT FROM WHERE", - "Failed to execute SQL", + "sqawk: Expected an expression, found: FROM", ) } @@ -1233,7 +1233,7 @@ fn err_order_by_all_rejected() -> Result<(), Box> { assert_query_fails( &["tests/data/employees.csv"], "SELECT name FROM employees ORDER BY ALL", - "not found", + "no such column: ALL", ) } @@ -1315,7 +1315,7 @@ fn aggregate_distinct_variants() -> Result<(), Box> { &["tests/data/repeats.csv"], "SELECT SUM(v), SUM(DISTINCT v), COUNT(v), COUNT(DISTINCT v), \ MIN(DISTINCT v), MAX(DISTINCT v), AVG(DISTINCT v) FROM repeats", - "SUM,SUM,COUNT,COUNT,MIN,MAX,AVG\n100,60,5,3,10,30,20", + "SUM(v),SUM(DISTINCT v),COUNT(v),COUNT(DISTINCT v),MIN(DISTINCT v),MAX(DISTINCT v),AVG(DISTINCT v)\n100,60,5,3,10,30,20.0", ) } @@ -1333,7 +1333,7 @@ fn case_as_projected_value() -> Result<(), Box> { assert_query( &["tests/data/employees.csv"], "SELECT name, CASE WHEN age > 30 THEN 'old' ELSE 'young' END FROM employees LIMIT 3", - "name,expr\nAlice,young\nBob,young\nCharlie,old", + "name,CASE WHEN age > 30 THEN 'old' ELSE 'young' END\nAlice,young\nBob,young\nCharlie,old", ) } @@ -1342,7 +1342,7 @@ fn cast_as_projected_value() -> Result<(), Box> { assert_query( &["tests/data/employees.csv"], "SELECT CAST(salary AS TEXT) FROM employees LIMIT 2", - "expr\n70000\n55000", + "CAST(salary AS TEXT)\n70000\n55000", ) } @@ -1351,7 +1351,7 @@ fn is_null_as_projected_value() -> Result<(), Box> { assert_query( &["tests/data/nullable.csv"], "SELECT id, score IS NULL FROM nullable", - "id,expr\n1,0\n2,1\n3,0\n4,0\n5,0", + "id,score IS NULL\n1,0\n2,1\n3,0\n4,0\n5,0", ) } @@ -1360,7 +1360,7 @@ fn like_as_projected_value() -> Result<(), Box> { assert_query( &["tests/data/employees.csv"], "SELECT name, name LIKE 'A%' FROM employees LIMIT 2", - "name,expr\nAlice,1\nBob,0", + "name,name LIKE 'A%'\nAlice,1\nBob,0", ) } @@ -1369,7 +1369,7 @@ fn between_as_projected_value() -> Result<(), Box> { assert_query( &["tests/data/employees.csv"], "SELECT name, age BETWEEN 30 AND 40 FROM employees LIMIT 3", - "name,expr\nAlice,1\nBob,0\nCharlie,1", + "name,age BETWEEN 30 AND 40\nAlice,1\nBob,0\nCharlie,1", ) } @@ -1414,7 +1414,7 @@ fn group_by_with_where_and_having() -> Result<(), Box> { HAVING COUNT(*) > 1", // age > 25 drops Bob(25) and Henry(22), leaving Marketing with one // row, which HAVING then removes. - "department,COUNT\nEngineering,3\nSales,2", + "department,COUNT(*)\nEngineering,3\nSales,2", ) } @@ -1425,7 +1425,7 @@ fn group_by_three_columns() -> Result<(), Box> { assert_query( &["tests/data/employees.csv"], "SELECT department, role, COUNT(*) FROM employees GROUP BY department, role", - "department,role,COUNT\n\ + "department,role,COUNT(*)\n\ Engineering,Developer,2\n\ Engineering,Manager,1\n\ HR,Intern,1\n\ @@ -1442,7 +1442,7 @@ fn group_by_with_where_excluding_a_whole_group() -> Result<(), Box 60000 GROUP BY department", - "department,COUNT\nEngineering,3\nSales,2", + "department,COUNT(*)\nEngineering,3\nSales,2", ) } @@ -1454,7 +1454,7 @@ fn having_on_a_group_key() -> Result<(), Box> { &["tests/data/employees.csv"], "SELECT department, COUNT(*) FROM employees GROUP BY department \ HAVING department = 'Sales'", - "department,COUNT\nSales,2", + "department,COUNT(*)\nSales,2", ) } @@ -1466,7 +1466,7 @@ fn having_selects_between_same_type_aggregates() -> Result<(), Box 90", - "department,SUM,SUM\nEngineering,210000,97", + "department,SUM(salary),SUM(age)\nEngineering,210000,97", ) } @@ -1481,7 +1481,7 @@ fn group_by_order_by_aggregate() -> Result<(), Box> { &["tests/data/employees.csv"], "SELECT department, COUNT(*) FROM employees GROUP BY department \ ORDER BY COUNT(*) DESC, department ASC", - "department,COUNT\nEngineering,3\nMarketing,2\nSales,2\nHR,1", + "department,COUNT(*)\nEngineering,3\nMarketing,2\nSales,2\nHR,1", ) } @@ -1491,7 +1491,7 @@ fn group_by_order_by_limit_together() -> Result<(), Box> &["tests/data/employees.csv"], "SELECT department, SUM(salary) FROM employees GROUP BY department \ ORDER BY SUM(salary) DESC LIMIT 2", - "department,SUM\nEngineering,210000\nSales,170000", + "department,SUM(salary)\nEngineering,210000\nSales,170000", ) } @@ -1501,7 +1501,7 @@ fn group_by_order_by_offset() -> Result<(), Box> { &["tests/data/employees.csv"], "SELECT department, COUNT(*) FROM employees GROUP BY department \ ORDER BY department ASC LIMIT 2 OFFSET 1", - "department,COUNT\nHR,1\nMarketing,2", + "department,COUNT(*)\nHR,1\nMarketing,2", ) } @@ -1511,7 +1511,7 @@ fn join_order_by_descending() -> Result<(), Box> { &["tests/data/users.csv", "tests/data/orders.csv"], "SELECT users.name, orders.id FROM users JOIN orders ON users.id = orders.user_id \ ORDER BY orders.id DESC", - "users.name,orders.id\nJohn,105\nJane,104\nJohn,103\nJane,102\nJohn,101", + "name,id\nJohn,105\nJane,104\nJohn,103\nJane,102\nJohn,101", ) } @@ -1521,7 +1521,7 @@ fn join_order_by_with_limit() -> Result<(), Box> { &["tests/data/users.csv", "tests/data/orders.csv"], "SELECT users.name, orders.id FROM users JOIN orders ON users.id = orders.user_id \ ORDER BY orders.id ASC LIMIT 2", - "users.name,orders.id\nJohn,101\nJane,102", + "name,id\nJohn,101\nJane,102", ) } @@ -1581,7 +1581,7 @@ fn aggregates_skip_nulls() -> Result<(), Box> { assert_query( &["tests/data/nullable.csv"], "SELECT COUNT(*), COUNT(score), SUM(score), AVG(score) FROM nullable", - "COUNT,COUNT,SUM,AVG\n5,4,300,75", + "COUNT(*),COUNT(score),SUM(score),AVG(score)\n5,4,300,75.0", ) } @@ -1611,7 +1611,7 @@ fn group_by_aggregate_before_key() -> Result<(), Box> { assert_query( &["tests/data/employees.csv"], "SELECT COUNT(*), SUM(salary), department FROM employees GROUP BY department", - "COUNT,SUM,department\n\ + "COUNT(*),SUM(salary),department\n\ 3,210000,Engineering\n\ 1,45000,HR\n\ 2,115000,Marketing\n\ @@ -1624,7 +1624,7 @@ fn group_by_key_repeated_in_projection() -> Result<(), Box Result<(), Box> { assert_query( &["tests/data/users.csv", "tests/data/orders.csv"], "SELECT orders.id, users.name FROM users JOIN orders ON users.id = orders.user_id", - "orders.id,users.name\n101,John\n103,John\n105,John\n102,Jane\n104,Jane", + "id,name\n101,John\n103,John\n105,John\n102,Jane\n104,Jane", ) } @@ -1663,7 +1663,7 @@ fn left_join_reversed_projection_nullfill() -> Result<(), Box Result<(), Box Result<(), Box Result<(), Box> { assert_query( &["tests/data/employees.csv"], "SELECT SUM(salary + age) FROM employees", - "SUM\n540257", + "SUM(salary + age)\n540257", ) } @@ -1755,7 +1755,7 @@ fn aggregate_over_function() -> Result<(), Box> { assert_query( &["tests/data/employees.csv"], "SELECT MAX(LENGTH(name)), MIN(LENGTH(name)) FROM employees", - "MAX,MIN\n7,3", + "MAX(LENGTH(name)),MIN(LENGTH(name))\n7,3", ) } @@ -1764,7 +1764,7 @@ fn arithmetic_over_aggregate() -> Result<(), Box> { assert_query( &["tests/data/employees.csv"], "SELECT SUM(salary) + 1 FROM employees", - "expr\n540001", + "SUM(salary) + 1\n540001", ) } @@ -1773,7 +1773,7 @@ fn arithmetic_between_two_aggregates() -> Result<(), Box> assert_query( &["tests/data/employees.csv"], "SELECT COUNT(*) * 2, MAX(salary) - MIN(salary) FROM employees", - "expr,expr\n16,45000", + "COUNT(*) * 2,MAX(salary) - MIN(salary)\n16,45000", ) } @@ -1782,7 +1782,7 @@ fn aggregate_mixed_with_plain_projection() -> Result<(), Box Result<(), Box> assert_query( &["tests/data/people.csv"], "SELECT name FROM people WHERE age > 30; SELECT COUNT(*) FROM people", - "name\nAlice\nCharlie\nCOUNT\n3", + "name\nAlice\nCharlie\nCOUNT(*)\n3", ) } @@ -1866,7 +1866,7 @@ fn unordered_window_gives_partition_totals() -> Result<(), Box Result<(), Box Result<(), Box Result<(), Box> { assert_query( &["tests/data/users.csv", "tests/data/orders.csv"], "SELECT COUNT(*) FROM users JOIN orders ON users.id = orders.user_id", - "COUNT\n5", + "COUNT(*)\n5", ) } @@ -1927,7 +1927,7 @@ fn group_by_over_inner_join() -> Result<(), Box> { &["tests/data/users.csv", "tests/data/orders.csv"], "SELECT users.name, COUNT(*) FROM users JOIN orders ON users.id = orders.user_id \ GROUP BY users.name", - "users.name,COUNT\nJane,2\nJohn,3", + "name,COUNT(*)\nJane,2\nJohn,3", ) } @@ -1937,7 +1937,7 @@ fn inner_join_and_comma_join_agree() -> Result<(), Box> { assert_query( &["tests/data/users.csv", "tests/data/orders.csv"], "SELECT COUNT(*) FROM users, orders WHERE users.id = orders.user_id", - "COUNT\n5", + "COUNT(*)\n5", ) } @@ -1950,7 +1950,7 @@ fn outer_join_aggregate_is_not_silently_rewritten() -> Result<(), Box Result<(), Box> { assert_query( &["tests/data/employees.csv"], "SELECT COUNT(*) FROM (SELECT name FROM employees WHERE department = 'Engineering') t", - "COUNT\n3", + "COUNT(*)\n3", ) } @@ -1985,7 +1985,7 @@ fn derived_table_over_group_by() -> Result<(), Box> { assert_query( &["tests/data/employees.csv"], "SELECT * FROM (SELECT department, COUNT(*) FROM employees GROUP BY department) t", - "department,COUNT\nEngineering,3\nHR,1\nMarketing,2\nSales,2", + "department,COUNT(*)\nEngineering,3\nHR,1\nMarketing,2\nSales,2", ) } @@ -2025,7 +2025,7 @@ fn derived_table_does_not_outlive_its_statement() -> Result<(), Box Result<(), Box> { &["tests/data/users.csv", "tests/data/orders.csv"], "SELECT users.name, orders.id FROM users JOIN orders \ ON users.id = orders.user_id AND orders.id > 102", - "users.name,orders.id\nJohn,103\nJohn,105\nJane,104", + "name,id\nJohn,103\nJohn,105\nJane,104", ) } @@ -2054,7 +2054,7 @@ fn join_on_with_or() -> Result<(), Box> { &["tests/data/users.csv", "tests/data/orders.csv"], "SELECT users.name, orders.id FROM users JOIN orders \ ON users.id = orders.user_id OR orders.id = 999", - "users.name,orders.id\nJohn,101\nJohn,103\nJohn,105\nJane,102\nJane,104", + "name,id\nJohn,101\nJohn,103\nJohn,105\nJane,102\nJane,104", ) } @@ -2066,7 +2066,7 @@ fn left_join_with_compound_on_keeps_nullfill() -> Result<(), Box 10", - "customers.name,purchases.item\nAnn,Pen\nBen,Desk\nCara,", + "name,item\nAnn,Pen\nBen,Desk\nCara,", ) } @@ -2084,7 +2084,7 @@ fn function_in_multi_table_where() -> Result<(), Box> { &["tests/data/users.csv", "tests/data/orders.csv"], "SELECT users.name, orders.id FROM users, orders \ WHERE users.id = orders.user_id AND UPPER(users.name) = 'JOHN'", - "users.name,orders.id\nJohn,101\nJohn,103\nJohn,105", + "name,id\nJohn,101\nJohn,103\nJohn,105", ) } @@ -2094,7 +2094,7 @@ fn arithmetic_in_multi_table_where() -> Result<(), Box> { &["tests/data/users.csv", "tests/data/orders.csv"], "SELECT users.name, orders.id FROM users, orders \ WHERE users.id = orders.user_id AND orders.id - 100 > 3", - "users.name,orders.id\nJohn,105\nJane,104", + "name,id\nJohn,105\nJane,104", ) } @@ -2104,7 +2104,7 @@ fn not_in_multi_table_where() -> Result<(), Box> { &["tests/data/users.csv", "tests/data/orders.csv"], "SELECT users.name, orders.id FROM users, orders \ WHERE users.id = orders.user_id AND NOT (orders.id = 101)", - "users.name,orders.id\nJohn,103\nJohn,105\nJane,102\nJane,104", + "name,id\nJohn,103\nJohn,105\nJane,102\nJane,104", ) } @@ -2115,7 +2115,7 @@ fn unqualified_column_in_multi_table_where() -> Result<(), Box Result<(), Box> { &["tests/data/users.csv", "tests/data/orders.csv"], "SELECT users.name, orders.id FROM users JOIN orders \ ON users.id = orders.user_id AND UPPER(users.name) = 'JANE'", - "users.name,orders.id\nJane,102\nJane,104", + "name,id\nJane,102\nJane,104", ) } @@ -2145,7 +2145,7 @@ fn three_table_join_with_compound_on() -> Result<(), Box> "SELECT users.name, products.name FROM users \ JOIN orders ON users.id = orders.user_id \ JOIN products ON orders.product_id = products.product_id AND products.price > 200", - "users.name,products.name\nJohn,Laptop\nJane,Phone\nJane,Monitor", + "name,name\nJohn,Laptop\nJane,Phone\nJane,Monitor", ) } @@ -2160,7 +2160,7 @@ fn three_table_join_unfiltered() -> Result<(), Box> { "SELECT users.name, products.name, products.price FROM users \ JOIN orders ON users.id = orders.user_id \ JOIN products ON orders.product_id = products.product_id", - "users.name,products.name,products.price\n\ + "name,name,price\n\ John,Laptop,1200\nJohn,Headphones,150\nJohn,Keyboard,80\n\ Jane,Phone,800\nJane,Monitor,350", ) @@ -2225,7 +2225,7 @@ fn alias_in_inner_join() -> Result<(), Box> { assert_query( &["tests/data/users.csv", "tests/data/orders.csv"], "SELECT u.name, o.id FROM users u JOIN orders o ON u.id = o.user_id", - "u.name,o.id\nJohn,101\nJohn,103\nJohn,105\nJane,102\nJane,104", + "name,id\nJohn,101\nJohn,103\nJohn,105\nJane,102\nJane,104", ) } @@ -2234,7 +2234,7 @@ fn alias_in_left_join() -> Result<(), Box> { assert_query( &["tests/data/customers.csv", "tests/data/purchases.csv"], "SELECT c.name, p.item FROM customers c LEFT JOIN purchases p ON c.id = p.customer_id", - "c.name,p.item\nAnn,Book\nAnn,Pen\nBen,Desk\nCara,", + "name,item\nAnn,Book\nAnn,Pen\nBen,Desk\nCara,", ) } @@ -2249,7 +2249,7 @@ fn alias_in_three_table_join() -> Result<(), Box> { "SELECT u.name, p.name FROM users u \ JOIN orders o ON u.id = o.user_id \ JOIN products p ON o.product_id = p.product_id", - "u.name,p.name\nJohn,Laptop\nJohn,Headphones\nJohn,Keyboard\nJane,Phone\nJane,Monitor", + "name,name\nJohn,Laptop\nJohn,Headphones\nJohn,Keyboard\nJane,Phone\nJane,Monitor", ) } @@ -2258,7 +2258,7 @@ fn alias_in_implicit_join() -> Result<(), Box> { assert_query( &["tests/data/users.csv", "tests/data/orders.csv"], "SELECT u.name, o.id FROM users u, orders o WHERE u.id = o.user_id", - "u.name,o.id\nJohn,101\nJohn,103\nJohn,105\nJane,102\nJane,104", + "name,id\nJohn,101\nJohn,103\nJohn,105\nJane,102\nJane,104", ) } @@ -2291,7 +2291,7 @@ fn grouped_join_order_by_and_limit() -> Result<(), Box> { &["tests/data/users.csv", "tests/data/orders.csv"], "SELECT u.name, COUNT(*) AS n FROM users u JOIN orders o ON u.id = o.user_id \ GROUP BY u.name ORDER BY n DESC LIMIT 1", - "u.name,n\nJohn,3", + "name,n\nJohn,3", ) } @@ -2303,7 +2303,7 @@ fn implicit_join_order_by_takes_top_rows() -> Result<(), Box Result<(), Box Result<(), Box> assert_query( &["tests/data/repeats_grouped.csv"], "SELECT grp, COUNT(DISTINCT val), SUM(DISTINCT val) FROM repeats_grouped GROUP BY grp", - "grp,COUNT,SUM\na,2,3\nb,2,4", + "grp,COUNT(DISTINCT val),SUM(DISTINCT val)\na,2,3\nb,2,4", ) } @@ -2345,7 +2345,7 @@ fn distinct_aggregate_in_having() -> Result<(), Box> { &["tests/data/repeats_grouped.csv"], "SELECT grp, COUNT(DISTINCT val) FROM repeats_grouped GROUP BY grp \ HAVING COUNT(DISTINCT val) > 1", - "grp,COUNT\na,2\nb,2", + "grp,COUNT(DISTINCT val)\na,2\nb,2", ) } @@ -2357,7 +2357,7 @@ fn order_by_over_select_star_join() -> Result<(), Box> { &["tests/data/customers.csv", "tests/data/purchases.csv"], "SELECT * FROM customers, purchases WHERE customers.id = purchases.customer_id \ ORDER BY purchases.id DESC", - "customers.id,customers.name,purchases.id,purchases.customer_id,purchases.item\n\ + "id,name,id,customer_id,item\n\ 2,Ben,12,2,Desk\n\ 1,Ann,11,1,Pen\n\ 1,Ann,10,1,Book", @@ -2370,7 +2370,7 @@ fn order_by_bare_key_against_qualified_projection() -> Result<(), Box Result<(), Box Result<(), Box Result<(), Box Result<(), Box> { + crate::helpers::assert_query_on( + &[crate::helpers::ONE], + "SELECT 1 IS 1 AS a, NULL IS NULL AS b, 1 IS NOT 2 AS c, NULL IS NULL / 2 AS d, \ + 'x' IS 'x' AND 0 AS e, -1 || 2 AS f, typeof(-1 || 2) AS g FROM one", + "a,b,c,d,e,f,g\n1,1,1,1,0,-12,text", + ) +} + +/// Parentheses add no depth (SQLite): `-(1+...+1)` with 999 terms is 1000 +/// levels, within the limit. +#[test] +fn parenthesized_999_term_chain_is_within_depth_limit() -> Result<(), Box> { + crate::helpers::assert_query_on( + &[crate::helpers::ONE], + &format!("SELECT -({}) AS v FROM one", vec!["1"; 999].join("+")), + "v\n-999", + ) +} + +// --------------------------------------------------------------------------- +// Value semantics: real rendering, load inference, the one comparison rule. +// --------------------------------------------------------------------------- + +/// A field loads as REAL exactly when it is the real's printed form +/// (SQLite's `CAST(x AS TEXT)`), so every cell prints back verbatim. +#[test] +fn real_inference_round_trips() -> Result<(), Box> { + assert_query_on( + &[( + "r.csv", + "v\n2.0\n0.30000000000000004\n1e5\n1.0e+20\n-0.0\n0.0001\n0.00001\n1.0e-05\n", + )], + "SELECT typeof(v) AS t, v FROM r", + "t,v\nreal,2.0\ntext,0.30000000000000004\ntext,1e5\nreal,1.0e+20\ntext,-0.0\n\ + real,0.0001\ntext,0.00001\nreal,1.0e-05", + ) +} + +/// `||` and CAST see the same text as output. +#[test] +fn real_text_form() -> Result<(), Box> { + assert_query_on( + &[ONE], + "SELECT 'a' || 2.0 AS a, CAST(0.1 + 0.2 AS TEXT) AS b, -0.0 AS c, 1e15 AS d, \ + 123456789012345.0 AS e, 1234567890123445.0 AS f FROM one", + "a,b,c,d,e,f\na2.0,0.3,0.0,1.0e+15,123456789012345.0,1.23456789012345e+15", + ) +} + +/// `=`, IN list, IN subquery, `<` and a hash join all use one rule; numbers +/// sort before other text. +#[test] +fn one_comparison_rule() -> Result<(), Box> { + let files = [("n.csv", "v\n10\n9\n007\n1e1\nabc\n-x\n")]; + assert_query_on( + &files, + "SELECT v, v = 10 AS eq, v IN (10, 7) AS inl, v IN (SELECT '10.0') AS ins, \ + v < '9.5' AS lt FROM n ORDER BY v", + "v,eq,inl,ins,lt\n007,0,1,0,1\n9,0,0,0,1\n10,1,1,1,0\n1e1,1,1,1,0\n\ + -x,0,0,0,0\nabc,0,0,0,0", + )?; + assert_query_on( + &files, + "SELECT a.v AS a, b.v AS b FROM n a JOIN n b ON a.v = b.v AND a.v || 'x' <> b.v || 'x' \ + ORDER BY 1, 2", + "a,b\n10,1e1\n1e1,10", + ) +} + +/// INTERSECT keeps the left side's value, UNION the last one inserted, as +/// SQLite does. +#[test] +fn compound_keeps_sqlite_value() -> Result<(), Box> { + assert_query_on(&[ONE], "SELECT 1 AS v INTERSECT SELECT 1.0", "v\n1")?; + assert_query_on(&[ONE], "SELECT 1 AS v UNION SELECT 1.0", "v\n1.0")?; + assert_query_on( + &[ONE], + "SELECT 1 AS v INTERSECT SELECT 1.0 UNION ALL SELECT 2.0 INTERSECT SELECT 2", + "v\n2.0", + ) +} + +// --------------------------------------------------------------------------- +// Name resolution: a qualifier must name a FROM source as it is known there +// (its alias when it has one); `main` is the only schema. +// --------------------------------------------------------------------------- + +const NAMES_D: (&str, &str) = ("d.csv", "k,n\n1,10\n2,20\n"); + +#[test] +fn qualifier_must_name_a_source() -> Result<(), Box> { + assert_query_on_fails(&[NAMES_D], "SELECT zz.k FROM d", "no such column: zz.k")?; + // An aliased table is known only by its alias, as in SQLite. + assert_query_on_fails( + &[NAMES_D], + "SELECT k FROM d x WHERE d.k = 1", + "no such column: d.k", + )?; + assert_query_on_fails(&[NAMES_D], "SELECT d.* FROM d x", "no such table: d")?; + assert_query_on_fails( + &[NAMES_D], + "DELETE FROM d WHERE zz.k = 1", + "no such column: zz.k", + )?; + assert_query_on(&[NAMES_D], "SELECT x.k FROM d x WHERE x.n = 20", "k\n2") +} + +#[test] +fn delete_target_alias_resolves() -> Result<(), Box> { + assert_query_on( + &[NAMES_D], + "DELETE FROM d AS x WHERE x.k = 1; SELECT * FROM d", + "k,n\n2,20", + )?; + assert_query_on( + &[NAMES_D], + "UPDATE d AS x SET n = x.n + 1 WHERE x.k = 2; SELECT * FROM d", + "k,n\n1,10\n2,21", + ) +} + +#[test] +fn main_schema_qualifier() -> Result<(), Box> { + assert_query_on( + &[NAMES_D], + "SELECT main.d.k, MAIN.d.n FROM main.d WHERE main.d.k = 2", + "k,n\n2,20", + )?; + assert_query_on(&[NAMES_D], "SELECT main.d.* FROM d", "k,n\n1,10\n2,20")?; + assert_query_on( + &[NAMES_D], + "UPDATE main.d SET n = 0 WHERE k = 1; SELECT n FROM d", + "n\n0\n20", + )?; + assert_query_on_fails(&[NAMES_D], "SELECT * FROM foo.d", "no such table: foo.d")?; + assert_query_on_fails( + &[NAMES_D], + "SELECT foo.d.k FROM d", + "no such column: foo.d.k", + )?; + assert_query_on_fails(&[NAMES_D], "SELECT foo.d.* FROM d", "no such table: foo.d")?; + assert_query_on_fails(&[NAMES_D], "DROP TABLE foo.d", "no such table: foo.d")?; + assert_query_on_fails( + &[NAMES_D], + "INSERT INTO foo.d VALUES (3, 30)", + "no such table: foo.d", + ) +} + +/// An aggregate whose arguments read only outer-query columns is rejected; +/// one that also reads an inner column aggregates in the subquery. +#[test] +fn outer_only_aggregate_rejected() -> Result<(), Box> { + let e = ("e.csv", "v\n5\n15\n"); + assert_query_on_fails( + &[NAMES_D, e], + "SELECT k, (SELECT MIN(d.n) FROM e) FROM d", + "aggregate of outer-query columns is not supported", + )?; + assert_query_on( + &[NAMES_D, e], + "SELECT k, (SELECT MIN(d.n + e.v) FROM e) AS m, \ + (SELECT COUNT(*) FROM e WHERE e.v > d.n) AS c FROM d", + "k,m,c\n1,15,1\n2,25,0", + ) +} + +// --------------------------------------------------------------------------- +// Result column names, as SQLite names them (expected names from SQLite +// 3.45 `cursor.description` / `PRAGMA table_info`). +// --------------------------------------------------------------------------- + +const NAMES_T: (&str, &str) = ("t.csv", "Id,Name\n1,ann\n"); +const NAMES_U: (&str, &str) = ("u.csv", "id,name\n1,bob\n"); + +#[test] +fn column_names_follow_sqlite() -> Result<(), Box> { + assert_query_on( + &[NAMES_T], + "SELECT ID, t.NAME, (id), -id, id + 1, id IS 5, id IS NOT DISTINCT FROM 5 FROM t", + "Id,Name,Id,-id,id + 1,id IS 5,id IS NOT DISTINCT FROM 5\n1,ann,1,-1,2,0,0", + )?; + assert_query_on( + &[NAMES_T, NAMES_U], + "SELECT * FROM t, u", + "Id,Name,id,name\n1,ann,1,bob", + )?; + assert_query_on( + &[NAMES_T, NAMES_U], + "SELECT t.*, u.name FROM t JOIN u USING (id)", + "Id,Name,name\n1,ann,bob", + )?; + assert_query_on( + &[NAMES_T], + "SELECT count(*), Count( * ), upper(min(name))||1 FROM t", + "count(*),Count( * ),upper(min(name))||1\n1,1,ANN1", + )?; + assert_query_on( + &[NAMES_T, NAMES_U], + "SELECT id + 0 FROM t UNION SELECT id AS k FROM u", + "id + 0\n1", + ) +} + +/// Derived-table and CREATE TABLE ... AS columns are made unique with `:N`; +/// a derived table names a column reference as written. +#[test] +fn derived_and_ctas_column_names_are_unique() -> Result<(), Box> { + assert_query_on( + &[NAMES_T], + "SELECT * FROM (SELECT id, ID, id + 1, id + 1 FROM t)", + "id,ID:1,id + 1,id + 1:1\n1,1,2,2", + )?; + assert_query_on( + &[NAMES_T, NAMES_U], + "SELECT * FROM (SELECT * FROM t, u)", + "Id,Name,id:1,name:1\n1,ann,1,bob", + )?; + assert_query_on( + &[NAMES_T], + "CREATE TABLE c AS SELECT id, t.id, name || ':' || '|' FROM t; SELECT * FROM c", + "Id,Id:1,name || ':' || '|'\n1,1,ann:|", + )?; + // Column names containing the instruction's own separators. + assert_query_on( + &[NAMES_T], + "CREATE TABLE z (\"a:b\" INT, \"c|d\" TEXT); INSERT INTO z VALUES (1, 'x'); \ + ALTER TABLE z ADD COLUMN \"e:f|g\" TEXT; SELECT * FROM z", + "a:b,c|d,e:f|g\n1,x,", + ) +} diff --git a/tests/headerless/mod.rs b/tests/headerless/mod.rs index b498ca6..00721c0 100644 --- a/tests/headerless/mod.rs +++ b/tests/headerless/mod.rs @@ -1,19 +1,16 @@ -//! Tests for headerless file auto-detection +//! Tests for headerless files (`--no-header`) //! -//! This file contains tests for automatic detection of files without headers -//! and the automatic generation of column names (a, b, c, ...). -//! -//! The guess applies only to non-.csv files: a `.csv` file's first line is -//! always its header, unless `--no-header` says the file has none. +//! The first line of every file is its header unless `--no-header` says the +//! file has none (or `--tabledef` names the columns); there is no guessing. +//! Headerless columns are named a, b, c, ... use crate::helpers::{create_custom_csv, create_temp_dir}; #[test] fn test_headerless_auto_detect_numeric_first_row() -> Result<(), Box> { - // Test auto-detection when first row contains numeric values + // A numeric first row is data under --no-header let temp_dir = create_temp_dir()?; - // First row has numbers - should be detected as data, not headers let data_file = create_custom_csv( temp_dir.path(), "numbers.txt", @@ -22,6 +19,7 @@ fn test_headerless_auto_detect_numeric_first_row() -> Result<(), Box Result<(), Box Result<(), Box> { - // Test auto-detection for /etc/passwd style files + // /etc/passwd style files: -F: --no-header let temp_dir = create_temp_dir()?; - // passwd-style: starts with "root" which triggers headerless detection let data_file = create_custom_csv( temp_dir.path(), "passwd", @@ -48,6 +45,7 @@ fn test_headerless_passwd_style() -> Result<(), Box> { let mut cmd = assert_cmd::cargo::cargo_bin_cmd!("sqawk"); cmd.arg("-F:") + .arg("--no-header") .arg("-s") .arg("SELECT a, c, f FROM passwd") .arg(data_file.to_str().unwrap()); @@ -62,10 +60,9 @@ fn test_headerless_passwd_style() -> Result<(), Box> { #[test] fn test_headerless_path_in_first_row() -> Result<(), Box> { - // Test auto-detection when first row contains a path + // A first row holding a path is data under --no-header let temp_dir = create_temp_dir()?; - // First field starts with "/" - should be detected as data let data_file = create_custom_csv( temp_dir.path(), "paths.txt", @@ -74,6 +71,7 @@ fn test_headerless_path_in_first_row() -> Result<(), Box> let mut cmd = assert_cmd::cargo::cargo_bin_cmd!("sqawk"); cmd.arg("-F,") + .arg("--no-header") .arg("-s") .arg("SELECT a, b FROM paths") .arg(data_file.to_str().unwrap()); @@ -95,6 +93,7 @@ fn test_headerless_alphabetic_columns() -> Result<(), Box let mut cmd = assert_cmd::cargo::cargo_bin_cmd!("sqawk"); cmd.arg("-F,") + .arg("--no-header") .arg("-s") .arg("SELECT a, b, c, d, e FROM data") .arg(data_file.to_str().unwrap()); @@ -120,6 +119,7 @@ fn test_headerless_filter_query() -> Result<(), Box> { let mut cmd = assert_cmd::cargo::cargo_bin_cmd!("sqawk"); cmd.arg("-F,") + .arg("--no-header") .arg("-s") .arg("SELECT b FROM data WHERE c > 75") .arg(data_file.to_str().unwrap()); @@ -145,6 +145,7 @@ fn test_headerless_aggregate() -> Result<(), Box> { let mut cmd = assert_cmd::cargo::cargo_bin_cmd!("sqawk"); cmd.arg("-F,") + .arg("--no-header") .arg("-s") .arg("SELECT b, SUM(c) FROM data GROUP BY b") .arg(data_file.to_str().unwrap()); @@ -159,7 +160,7 @@ fn test_headerless_aggregate() -> Result<(), Box> { #[test] fn test_headerless_with_comments() -> Result<(), Box> { - // Test headerless file with comment lines + // Headerless file with comment lines (--comments) let temp_dir = create_temp_dir()?; let data_file = create_custom_csv( @@ -170,6 +171,8 @@ fn test_headerless_with_comments() -> Result<(), Box> { let mut cmd = assert_cmd::cargo::cargo_bin_cmd!("sqawk"); cmd.arg("-F,") + .arg("--no-header") + .arg("--comments") .arg("-s") .arg("SELECT * FROM data") .arg(data_file.to_str().unwrap()); @@ -184,13 +187,14 @@ fn test_headerless_with_comments() -> Result<(), Box> { #[test] fn test_headerless_asterisk_field() -> Result<(), Box> { - // Test auto-detection when field contains asterisk (like passwd x field) + // A field holding an asterisk (like the passwd x field) let temp_dir = create_temp_dir()?; let data_file = create_custom_csv(temp_dir.path(), "data.txt", "user1,*,1001\nuser2,*,1002\n")?; let mut cmd = assert_cmd::cargo::cargo_bin_cmd!("sqawk"); cmd.arg("-F,") + .arg("--no-header") .arg("-s") .arg("SELECT a, c FROM data") .arg(data_file.to_str().unwrap()); @@ -232,7 +236,7 @@ fn test_no_header_writeback_adds_no_header() -> Result<(), Box Result<(), Box> { +fn test_colon_no_header_writeback_adds_no_header() -> Result<(), Box> { let temp_dir = create_temp_dir()?; let data_file = create_custom_csv( temp_dir.path(), @@ -242,7 +246,7 @@ fn test_guessed_headerless_writeback_adds_no_header() -> Result<(), Box Result<(), Box> { // Check output - should have combined columns for a complete cross join cmd.assert() .success() - .stdout(predicate::str::contains("users.id,users.name,users.email,orders.id,orders.user_id,orders.product_id,orders.date")) - .stderr(predicate::str::contains("Processing multiple tables in FROM clause as CROSS JOINs")); + .stdout(predicate::str::contains( + "id,name,email,id,user_id,product_id,date", + )) + .stderr(predicate::str::contains( + "Processing multiple tables in FROM clause as CROSS JOINs", + )); // Verify we have a proper cross join with both users appearing in the result cmd.assert() @@ -71,14 +75,26 @@ fn test_inner_join() -> Result<(), Box> { // Check output - should include john and jane, but only their own orders cmd.assert() .success() - .stdout(predicate::str::contains("users.id,users.name,users.email,orders.id,orders.user_id,orders.product_id,orders.date")) + .stdout(predicate::str::contains( + "id,name,email,id,user_id,product_id,date", + )) // John's orders - .stdout(predicate::str::contains("1,John,john@example.com,101,1,101,2023-01-15")) - .stdout(predicate::str::contains("1,John,john@example.com,103,1,103,2023-02-10")) - .stdout(predicate::str::contains("1,John,john@example.com,105,1,104,2023-03-05")) + .stdout(predicate::str::contains( + "1,John,john@example.com,101,1,101,2023-01-15", + )) + .stdout(predicate::str::contains( + "1,John,john@example.com,103,1,103,2023-02-10", + )) + .stdout(predicate::str::contains( + "1,John,john@example.com,105,1,104,2023-03-05", + )) // Jane's orders - .stdout(predicate::str::contains("2,Jane,jane@example.com,102,2,102,2023-01-20")) - .stdout(predicate::str::contains("2,Jane,jane@example.com,104,2,105,2023-02-25")); + .stdout(predicate::str::contains( + "2,Jane,jane@example.com,102,2,102,2023-01-20", + )) + .stdout(predicate::str::contains( + "2,Jane,jane@example.com,104,2,105,2023-02-25", + )); // Verify verbose output contains filtering indications cmd.assert() @@ -100,9 +116,9 @@ fn test_inner_join_with_projection() -> Result<(), Box> { .arg(get_orders_file().to_str().unwrap()); // Check output - should include only selected columns - cmd.assert().success().stdout(predicate::str::contains( - "users.name,orders.product_id,orders.date", - )); + cmd.assert() + .success() + .stdout(predicate::str::contains("name,product_id,date")); // Verify we have the correct data cmd.assert() @@ -123,9 +139,9 @@ fn test_three_table_join() -> Result<(), Box> { .arg(get_products_file().to_str().unwrap()); // Check output headers - cmd.assert().success().stdout(predicate::str::contains( - "users.name,products.name,products.price,orders.date", - )); + cmd.assert() + .success() + .stdout(predicate::str::contains("name,name,price,date")); // Check for some expected combinations cmd.assert() @@ -150,9 +166,7 @@ fn test_join_with_additional_filtering() -> Result<(), Box 500 cmd.assert() .success() - .stdout(predicate::str::contains( - "users.name,products.name,products.price", - )) + .stdout(predicate::str::contains("name,name,price")) // These should be included (price > 500) .stdout(predicate::str::contains("John,Laptop,1200")) .stdout(predicate::str::contains("Jane,Phone,800")); @@ -190,7 +204,7 @@ fn test_join_order_preservation() -> Result<(), Box> { // Just check that we get the expected results in some order cmd.assert() .success() - .stdout(predicate::str::contains("users.name,orders.id")) + .stdout(predicate::str::contains("name,id")) .stdout(predicate::str::contains("John,101")) .stdout(predicate::str::contains("Jane,102")); diff --git a/tests/join/planner.rs b/tests/join/planner.rs index 95ae6f4..fb4ff8d 100644 --- a/tests/join/planner.rs +++ b/tests/join/planner.rs @@ -141,7 +141,7 @@ fn on_clause_cannot_reference_a_later_table() -> TestResult { fn right_join_after_outer_join_is_unsupported() -> TestResult { assert_on_fails( "SELECT lt.a FROM lt LEFT JOIN rt ON lt.k = rt.k RIGHT JOIN xt ON xt.k = lt.k", - "Unsupported", + "not supported", ) } diff --git a/tests/join_on/mod.rs b/tests/join_on/mod.rs index 82d3d03..17ca5ed 100644 --- a/tests/join_on/mod.rs +++ b/tests/join_on/mod.rs @@ -17,9 +17,9 @@ fn test_inner_join_on_basic() -> Result<(), Box> { .arg("-v"); // Check output - cmd.assert().success().stdout(predicate::str::contains( - "users.name,orders.product_id,orders.date", - )); + cmd.assert() + .success() + .stdout(predicate::str::contains("name,product_id,date")); // Verify join results cmd.assert() @@ -58,9 +58,9 @@ fn test_inner_join_on_with_where() -> Result<(), Box> { .arg("-v"); // Check output - should only include orders with product_id > 102 - cmd.assert().success().stdout(predicate::str::contains( - "users.name,orders.product_id,orders.date", - )); + cmd.assert() + .success() + .stdout(predicate::str::contains("name,product_id,date")); // Verify results match the filter cmd.assert() @@ -95,9 +95,9 @@ fn test_three_way_inner_join_on() -> Result<(), Box> { .arg("-v"); // Check column aliases and content - cmd.assert().success().stdout(predicate::str::contains( - "user,product,products.price,orders.date", - )); + cmd.assert() + .success() + .stdout(predicate::str::contains("user,product,price,date")); // Check some expected join results cmd.assert() diff --git a/tests/math_functions/mod.rs b/tests/math_functions/mod.rs index cd55f2f..69233a1 100644 --- a/tests/math_functions/mod.rs +++ b/tests/math_functions/mod.rs @@ -98,7 +98,7 @@ fn test_round_precision() -> Result<(), Box> { &["tests/data/employees.csv"], "SELECT ROUND(3.14159, 2) AS a, ROUND(2.71828, 3) AS b, ROUND(1.005, 0) AS c \ FROM employees LIMIT 1", - concat!("a,b,c", "\n3.14,2.718,1"), + concat!("a,b,c", "\n3.14,2.718,1.0"), ) } @@ -108,17 +108,17 @@ fn test_round_precision_zero_matches_one_arg() -> Result<(), Box Result<(), Box> { crate::helpers::assert_query( &["tests/data/employees.csv"], "SELECT ROUND(1234.5, -2) AS a FROM employees LIMIT 1", - concat!("a", "\n1200"), + concat!("a", "\n1235.0"), ) } @@ -130,7 +130,7 @@ fn test_round_precision_on_column() -> Result<(), Box> { "SELECT name, ROUND(salary / 7.0, 2) AS share FROM employees LIMIT 3", concat!( "name,share", - "\nAlice,10000", + "\nAlice,10000.0", "\nBob,7857.14", "\nCharlie,9285.71", ), @@ -177,7 +177,7 @@ fn test_round_precision_without_from() -> Result<(), Box> crate::helpers::assert_query( &["tests/data/employees.csv"], "SELECT ROUND(3.14159, 2) AS a, ROUND(1234.5, -2) AS b", - concat!("a,b", "\n3.14,1200"), + concat!("a,b", "\n3.14,1235.0"), ) } @@ -219,7 +219,7 @@ fn test_round_precision_distinguishes_column_names() -> Result<(), Box>; + + const T: (&str, &str) = ("t.csv", "a,b\n3,x\n1,y\n2,z\n"); + const U: (&str, &str) = ("u.csv", "c\n5\n1\n"); + + #[test] + fn parenthesized_and_signed_positions() -> R { + for sql in [ + "SELECT a FROM t ORDER BY (1)", + "SELECT a FROM t ORDER BY +(+1)", + "SELECT a FROM t ORDER BY - -1", + "SELECT a FROM t ORDER BY 01", + "SELECT a FROM t ORDER BY 0x1", + ] { + assert_query_on(&[T], sql, "a\n1\n2\n3")?; + } + assert_query_on( + &[T], + "SELECT b, a FROM t ORDER BY 0x2", + "b,a\ny,1\nz,2\nx,3", + ) + } + + #[test] + fn non_positions_are_constants() -> R { + for sql in [ + "SELECT a FROM t ORDER BY 1e0", + "SELECT a FROM t ORDER BY 1+0", + "SELECT a FROM t ORDER BY '1'", + "SELECT a FROM t ORDER BY -2147483648", + "SELECT a FROM t ORDER BY 0x80000000", + "SELECT a FROM t ORDER BY 0xFFFFFFFFFFFFFFFF", + ] { + assert_query_on(&[T], sql, "a\n3\n1\n2")?; + } + assert_query_on( + &[T], + "SELECT COUNT(*) AS n FROM t GROUP BY -2147483648", + "n\n3", + )?; + assert_query_on(&[T], "SELECT COUNT(*) AS n FROM t GROUP BY 1.0", "n\n3") + } + + #[test] + fn out_of_range_errors_name_the_term() -> R { + for (sql, err) in [ + ( + "SELECT a FROM t ORDER BY -(1)", + "1st ORDER BY term out of range - should be between 1 and 1", + ), + ( + "SELECT a FROM t ORDER BY +-1", + "1st ORDER BY term out of range - should be between 1 and 1", + ), + ( + "SELECT a FROM t ORDER BY 0x7fffffff", + "1st ORDER BY term out of range - should be between 1 and 1", + ), + ( + "SELECT a FROM t ORDER BY 1,1,1,1,1,1,1,1,1,1,1,2", + "12th ORDER BY term out of range - should be between 1 and 1", + ), + ( + "SELECT a FROM t ORDER BY 1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,2", + "22nd ORDER BY term out of range - should be between 1 and 1", + ), + ( + "SELECT a FROM t GROUP BY a ORDER BY 2", + "1st ORDER BY term out of range - should be between 1 and 1", + ), + ( + "SELECT a, b FROM t GROUP BY 1, 3", + "2nd GROUP BY term out of range - should be between 1 and 2", + ), + ] { + assert_query_on_fails(&[T], sql, err)?; + } + Ok(()) + } + + #[test] + fn grouped_positions() -> R { + assert_query_on( + &[T], + "SELECT a % 2 AS p, COUNT(*) AS n FROM t GROUP BY (+1) ORDER BY 1", + "p,n\n0,1\n1,2", + )?; + assert_query_on( + &[T], + "SELECT a AS x FROM t GROUP BY x HAVING 1 ORDER BY +1 DESC", + "x\n3\n2\n1", + ) + } + + #[test] + fn compound_positions() -> R { + assert_query_on( + &[T, U], + "SELECT a FROM t UNION SELECT c FROM u ORDER BY +1 DESC", + "a\n5\n3\n2\n1", + )?; + assert_query_on( + &[T, U], + "SELECT a FROM t UNION ALL SELECT c FROM u ORDER BY (1)", + "a\n1\n1\n2\n3\n5", + )?; + assert_query_on( + &[T, U], + "SELECT a, b FROM t UNION SELECT c, 'w' FROM u ORDER BY - -2, 1", + "a,b\n1,w\n5,w\n3,x\n1,y\n2,z", + ) + } + + /// In a compound, a term that is not a position must name a result + /// column; a constant does not. + #[test] + fn compound_non_positions_match_no_column() -> R { + for sql in [ + "SELECT a FROM t UNION SELECT c FROM u ORDER BY 1.0", + "SELECT a FROM t UNION SELECT c FROM u ORDER BY 4294967297", + "SELECT a FROM t UNION SELECT c FROM u ORDER BY 2147483648", + "SELECT a FROM t UNION SELECT c FROM u ORDER BY 1+0", + ] { + assert_query_on_fails( + &[T, U], + sql, + "1st ORDER BY term does not match any column in the result set", + )?; + } + for (sql, err) in [ + ( + "SELECT a FROM t UNION SELECT c FROM u ORDER BY -1", + "1st ORDER BY term out of range - should be between 1 and 1", + ), + ( + "SELECT a FROM t UNION SELECT c FROM u ORDER BY 0", + "1st ORDER BY term out of range - should be between 1 and 1", + ), + ( + "SELECT a, b FROM t UNION SELECT c, 'w' FROM u ORDER BY 1, 3", + "2nd ORDER BY term out of range - should be between 1 and 2", + ), + ] { + assert_query_on_fails(&[T, U], sql, err)?; + } + Ok(()) + } + + #[test] + fn distinct_positions() -> R { + assert_query_on( + &[T], + "SELECT DISTINCT a % 2 AS p FROM t ORDER BY -(-1) DESC", + "p\n1\n0", + )?; + assert_query_on( + &[T], + "SELECT DISTINCT a % 2 AS p FROM t ORDER BY 1.0", + "p\n1\n0", + )?; + assert_query_on_fails( + &[T], + "SELECT DISTINCT a FROM t ORDER BY 0", + "1st ORDER BY term out of range - should be between 1 and 1", + ) + } + + #[test] + fn subquery_positions() -> R { + assert_query_on( + &[T], + "SELECT * FROM (SELECT a FROM t ORDER BY +1 DESC LIMIT 2)", + "a\n3\n2", + )?; + assert_query_on( + &[T], + "SELECT (SELECT b FROM t ORDER BY (1) DESC LIMIT 1) AS v", + "v\nz", + )?; + assert_query_on( + &[T], + "SELECT (SELECT b FROM t ORDER BY 4294967297 LIMIT 1) AS v", + "v\nx", + )?; + for sql in [ + "SELECT a FROM t WHERE a IN (SELECT a FROM t ORDER BY 2)", + "SELECT a FROM t WHERE EXISTS (SELECT 1 FROM t ORDER BY -1)", + ] { + assert_query_on_fails( + &[T], + sql, + "1st ORDER BY term out of range - should be between 1 and 1", + )?; + } + Ok(()) + } + + /// A window's ORDER BY / PARTITION BY integer is a constant, never a + /// position: every row is a peer. + #[test] + fn window_integers_are_constants() -> R { + for sql in [ + "SELECT a, RANK() OVER (ORDER BY 1) AS r FROM t ORDER BY a", + "SELECT a, RANK() OVER (ORDER BY -1) AS r FROM t ORDER BY a", + "SELECT a, RANK() OVER (PARTITION BY 2 ORDER BY 5) AS r FROM t ORDER BY 1", + ] { + assert_query_on(&[T], sql, "a,r\n1,1\n2,1\n3,1")?; + } + assert_query_on( + &[T], + "SELECT a, SUM(a) OVER (ORDER BY 1) AS s FROM t ORDER BY a", + "a,s\n1,6\n2,6\n3,6", + ) + } +} diff --git a/tests/repl/commands.rs b/tests/repl/commands.rs index cde042e..ba8abc5 100644 --- a/tests/repl/commands.rs +++ b/tests/repl/commands.rs @@ -10,14 +10,8 @@ fn test_repl_basic_dot_commands() { let test_commands = ".help\n.version\n.tables\n.cd .\n.changes on\n.changes off\n.exit\n"; // Start the sqawk process with sample data loaded - let mut process = Command::new("cargo") - .args([ - "run", - "--bin", - "sqawk", - "tests/data/sample.csv", - "--interactive", - ]) + let mut process = Command::new(env!("CARGO_BIN_EXE_sqawk")) + .args(["tests/data/sample.csv", "--interactive"]) .stdin(Stdio::piped()) .stdout(Stdio::piped()) .stderr(Stdio::piped()) @@ -49,14 +43,8 @@ fn test_repl_show_command() { let test_commands = ".show\n.show tables\n.exit\n"; // Start the sqawk process with sample data loaded - let mut process = Command::new("cargo") - .args([ - "run", - "--bin", - "sqawk", - "tests/data/sample.csv", - "--interactive", - ]) + let mut process = Command::new(env!("CARGO_BIN_EXE_sqawk")) + .args(["tests/data/sample.csv", "--interactive"]) .stdin(Stdio::piped()) .stdout(Stdio::piped()) .stderr(Stdio::piped()) @@ -89,14 +77,8 @@ fn test_repl_stats_command() { ".stats on\nSELECT * FROM sample;\n.stats off\nSELECT * FROM sample;\n.exit\n"; // Start the sqawk process with sample data loaded - let mut process = Command::new("cargo") - .args([ - "run", - "--bin", - "sqawk", - "tests/data/sample.csv", - "--interactive", - ]) + let mut process = Command::new(env!("CARGO_BIN_EXE_sqawk")) + .args(["tests/data/sample.csv", "--interactive"]) .stdin(Stdio::piped()) .stdout(Stdio::piped()) .stderr(Stdio::piped()) @@ -256,9 +238,11 @@ fn test_repl_reports_changes_for_trailing_dml() { ["1 rows affected"], "the DELETE must report once and the SELECTs not at all, got:\n{stdout}" ); - assert!( - stdout.contains("Query returned 3 rows") && stdout.contains("Query returned 2 rows"), - "both SELECT result sets should print, got:\n{stdout}" + // Both SELECT result sets print; a piped session's stdout carries + // nothing else. + assert_eq!( + stdout, + "id,name,age\n1,Alice,32\n2,Bob,25\n3,Charlie,35\n1 rows affected\nid\n2\n3\n" ); } diff --git a/tests/repl/mod.rs b/tests/repl/mod.rs index 550099e..2851b65 100644 --- a/tests/repl/mod.rs +++ b/tests/repl/mod.rs @@ -14,14 +14,8 @@ fn test_repl_basic_commands() { let test_commands = ".help\n.tables\nSELECT * FROM sample;\n.exit\n"; // Start the sqawk process with sample data loaded - let mut process = Command::new("cargo") - .args([ - "run", - "--bin", - "sqawk", - "tests/data/sample.csv", - "--interactive", - ]) + let mut process = Command::new(env!("CARGO_BIN_EXE_sqawk")) + .args(["tests/data/sample.csv", "--interactive"]) .stdin(Stdio::piped()) .stdout(Stdio::piped()) .spawn() @@ -52,14 +46,8 @@ fn test_repl_table_operations() { let test_commands = ".tables\nSELECT COUNT(*) FROM sample;\nUPDATE sample SET age = 31 WHERE id = 1;\nSELECT * FROM sample;\n.exit\n"; // Start the sqawk process with sample data loaded - let mut process = Command::new("cargo") - .args([ - "run", - "--bin", - "sqawk", - "tests/data/sample.csv", - "--interactive", - ]) + let mut process = Command::new(env!("CARGO_BIN_EXE_sqawk")) + .args(["tests/data/sample.csv", "--interactive"]) .stdin(Stdio::piped()) .stdout(Stdio::piped()) .spawn() @@ -98,14 +86,8 @@ fn test_repl_write_toggle() { ".write\nUPDATE sample SET age = 32 WHERE id = 1;\n.write\nSELECT * FROM sample;\n.exit\n"; // Start the sqawk process with temp data loaded - let mut process = Command::new("cargo") - .args([ - "run", - "--bin", - "sqawk", - temp_file_path.to_str().unwrap(), - "--interactive", - ]) + let mut process = Command::new(env!("CARGO_BIN_EXE_sqawk")) + .args([temp_file_path.to_str().unwrap(), "--interactive"]) .stdin(Stdio::piped()) .stdout(Stdio::piped()) .spawn() @@ -135,14 +117,8 @@ fn test_repl_multiline_statements() { "SELECT\n id,\n name,\n age\nFROM\n sample\nWHERE\n age > 25;\n.exit\n"; // Start the sqawk process with sample data loaded - let mut process = Command::new("cargo") - .args([ - "run", - "--bin", - "sqawk", - "tests/data/sample.csv", - "--interactive", - ]) + let mut process = Command::new(env!("CARGO_BIN_EXE_sqawk")) + .args(["tests/data/sample.csv", "--interactive"]) .stdin(Stdio::piped()) .stdout(Stdio::piped()) .spawn() @@ -186,14 +162,8 @@ fn test_repl_save_command() -> Result<(), Box> { "UPDATE sample SET age = 33 WHERE id = 1;\n.save\nUPDATE sample SET age = 34 WHERE id = 2;\n.save sample\n.exit\n"; // Start the sqawk process with the temporary file - let mut process = Command::new("cargo") - .args([ - "run", - "--bin", - "sqawk", - temp_file_path.to_str().unwrap(), - "--interactive", - ]) + let mut process = Command::new(env!("CARGO_BIN_EXE_sqawk")) + .args([temp_file_path.to_str().unwrap(), "--interactive"]) .stdin(Stdio::piped()) .stdout(Stdio::piped()) .spawn() diff --git a/tests/sqlite_diff/harness.py b/tests/sqlite_diff/harness.py deleted file mode 100644 index d064054..0000000 --- a/tests/sqlite_diff/harness.py +++ /dev/null @@ -1,205 +0,0 @@ -#!/usr/bin/env python3 -"""Differential tester: sqawk vs sqlite3. - -Usage: python3 -I tests/sqlite_diff/harness.py [queries_file ...] -Set SQAWK to override the binary (default: target/release/sqawk; run -`cargo build --release` first). Not part of `cargo test`. -Each query file: one test per line (blank / '--' lines skipped). -Prefix 'O:' => compare ordered; otherwise multiset compare. -Multiple statements separated by ';' are allowed (DML); only the -final SELECT's result is compared (sqlite runs all, collects last). -""" -import csv, io, os, re, sqlite3, subprocess, sys, tempfile, time, collections - -HERE = os.path.dirname(os.path.abspath(__file__)) -REPO = os.path.dirname(os.path.dirname(HERE)) -SQAWK = os.environ.get("SQAWK", os.path.join(REPO, "target", "release", "sqawk")) -DATA = tempfile.mkdtemp(prefix="sqawk-diff-") - -TABLES = { - "emp": (["id", "name", "dept", "salary", "mgr", "bonus"], [ - [1, "Alice", "Eng", 100, None, 10.5], - [2, "Bob", "Eng", 80, 1, None], - [3, "Carol", "Sales", 70, 1, 5], - [4, "Dave", "Sales", 70, 3, -2.25], - [5, "Eve", None, 50, 3, 0], - [6, "Frank", "HR", None, 1, None], - [7, "alice", "Eng", 100, 2, 3], - [8, "Grace", "Ops", -10, None, 1.5], - [9, "Heidi", "HR", 60, 6, None], - [10, "Bob", "Eng", 80, 1, 2], - ]), - "dept": (["dept", "dname", "budget"], [ - ["Eng", "Engineering", 1000], - ["Sales", "Sales Team", 500.5], - ["HR", "Human Res", None], - ["Mkt", "Marketing", 200], - ]), - "t1": (["k", "a", "s"], [ - [1, 10, "x"], [2, None, "y"], [3, 30, None], [3, 30, None], [4, -5, "x%y"], [5, 0, "A_b"], - ]), - "t2": (["k", "b"], [ - [1, 100], [3, 300], [3, 301], [6, 600], [None, 999], - ]), - "t3": (["k", "c"], [ - [1, "one"], [6, "six"], [7, "seven"], - ]), - "e": (["x"], []), # empty table (header only) - "uni": (["id", "w"], [ - [1, "héllo"], [2, "日本語"], [3, "naïve café"], [4, " pad "], [5, ""], [6, "ABC"], - ]), -} - - -def infer(v): - if v is None or v == "": - return None - if isinstance(v, (int, float)): - return v - return v - - -def write_data(): - os.makedirs(DATA, exist_ok=True) - for t, (cols, rows) in TABLES.items(): - with open(os.path.join(DATA, t + ".csv"), "w", newline="") as f: - w = csv.writer(f, lineterminator="\n") - w.writerow(cols) - for r in rows: - w.writerow(["" if x is None else x for x in r]) - - -def sqlite_db(): - db = sqlite3.connect(":memory:") - for t, (cols, rows) in TABLES.items(): - db.execute("CREATE TABLE %s (%s)" % (t, ",".join(cols))) - db.executemany("INSERT INTO %s VALUES (%s)" % (t, ",".join("?" * len(cols))), - [[infer(x) for x in r] for r in rows]) - return db - - -def norm(x): - if x is None: - return "NULL" - if isinstance(x, bool): - return str(int(x)) - if isinstance(x, int): - return str(x) - if isinstance(x, float): - if x != x: - return "NaN" - if x == int(x) and abs(x) < 1e15: - return str(int(x)) - return "%.9g" % x - s = str(x) - if s == "NULL": - return "NULL" - if re.fullmatch(r"-?\d+(\.\d+)?([eE][-+]?\d+)?", s): - try: - return norm(float(s)) if not re.fullmatch(r"-?\d+", s) else str(int(s)) - except ValueError: - pass - return s - - -def run_sqlite(db, sql): - db.execute("SAVEPOINT sp") - try: - last = None - for stmt in split_stmts(sql): - cur = db.execute(stmt) - if cur.description: - last = [tuple(norm(v) for v in row) for row in cur.fetchall()] - return ("ok", last) - except Exception as ex: - return ("err", str(ex)) - finally: - db.execute("ROLLBACK TO sp") - db.execute("RELEASE sp") - - -def split_stmts(sql): - out, cur, q = [], "", False - for ch in sql: - if ch == "'": - q = not q - if ch == ";" and not q: - if cur.strip(): - out.append(cur) - cur = "" - else: - cur += ch - if cur.strip(): - out.append(cur) - return out - - -def tables_in(sql): - return [t for t in TABLES if re.search(r"\b%s\b" % t, sql, re.IGNORECASE)] - - -def run_sqawk(sql, extra_args=()): - files = [os.path.join(DATA, t + ".csv") for t in tables_in(sql)] or [os.path.join(DATA, "t1.csv")] - t0 = time.time() - try: - p = subprocess.run([SQAWK, *extra_args, "-s", sql, *files], capture_output=True, - text=True, timeout=10) - except subprocess.TimeoutExpired: - return ("hang", None, 10) - dt = time.time() - t0 - if p.returncode != 0: - kind = "panic" if ("panicked" in p.stderr or p.returncode >= 100 or p.returncode < 0) else "err" - return (kind, p.stderr.strip().splitlines()[-1] if p.stderr.strip() else "rc=%d" % p.returncode, dt) - lines = list(csv.reader(io.StringIO(p.stdout))) - # result sets each begin with a header; keep the last one. A header is - # hard to detect generically, so for multi-statement SQL we rely on the - # final SELECT being the only SELECT. - # sqawk renders NULL as an empty field - rows = [tuple("NULL" if v == "" else norm(v) for v in r) for r in lines[1:]] - return ("ok", rows, dt) - - -def compare(ordered, a, b): - if ordered: - return a == b - return collections.Counter(a) == collections.Counter(b) - - -def main(): - write_data() - db = sqlite_db() - files = sys.argv[1:] or [os.path.join(HERE, "queries.sql")] - stats = collections.Counter() - for qf in files: - for line in open(qf, encoding="utf-8"): - q = line.strip() - if not q or q.startswith("--"): - continue - ordered = q.startswith("O:") - if ordered: - q = q[2:].strip() - sk = run_sqawk(q) - sl = run_sqlite(db, q) - if sk[0] in ("hang", "panic"): - stats[sk[0]] += 1 - print("%s: %s\n %s" % (sk[0].upper(), q, sk[1])) - elif sk[0] == "err": - stats["sqawk_err"] += 1 - print("UNSUPPORTED/ERR: %s\n sqawk: %s\n sqlite: %s" % ( - q, sk[1], "ok" if sl[0] == "ok" else sl[1])) - elif sl[0] == "err": - stats["sqlite_err"] += 1 - print("SQLITE-ERR(sqawk ok): %s\n sqlite: %s\n sqawk: %s" % (q, sl[1], sk[1][:8])) - elif compare(ordered, sk[1], sl[1] or []): - stats["pass"] += 1 - else: - stats["DIFF"] += 1 - print("DIFF%s: %s\n sqawk : %s\n sqlite: %s" % ( - "(ordered)" if ordered else "", q, sk[1][:12], (sl[1] or [])[:12])) - if sk[0] == "ok" and sk[2] > 2: - print("SLOW %.1fs: %s" % (sk[2], q)) - print("\nSUMMARY", dict(stats)) - - -if __name__ == "__main__": - main() diff --git a/tests/stdin/mod.rs b/tests/stdin/mod.rs index 7818973..8028772 100644 --- a/tests/stdin/mod.rs +++ b/tests/stdin/mod.rs @@ -44,7 +44,7 @@ fn named_stdin_table() { let (stdout, stderr, ok) = run_with_stdin(&["-s", "SELECT COUNT(*) FROM emp", "emp=-"], EMPLOYEES); assert!(ok, "sqawk failed: {stderr}"); - assert_eq!(stdout, "COUNT\n3"); + assert_eq!(stdout, "COUNT(*)\n3"); } /// Piped input is comma-separated by default, so sqawk's own output can be @@ -81,7 +81,7 @@ fn stdin_joins_with_a_file() { EMPLOYEES, ); assert!(ok, "sqawk failed: {stderr}"); - assert_eq!(stdout, "s.name\nAlice\nBob"); + assert_eq!(stdout, "name\nAlice\nBob"); } /// --tabledef supplies column names for headerless piped input. diff --git a/tests/string_functions_ext/mod.rs b/tests/string_functions_ext/mod.rs index d83c40d..0f8b106 100644 --- a/tests/string_functions_ext/mod.rs +++ b/tests/string_functions_ext/mod.rs @@ -202,6 +202,37 @@ mod sqlite_semantics { ) } + /// The rest of SQLite's printf: flags `# , !`, `*` width and precision, + /// `%q %Q %w %c %r %u %.Nd`, real rounding to 16 digits, and NULL for a + /// format that produces nothing. + #[test] + fn printf_sqlite_conversions() -> R { + assert_query_on( + &[ONE], + "SELECT printf('%#x|%#o|%,d|%.3d|%u', 255, 8, -1234567, 7, -1) AS a, \ + printf('%*d|%-*d|%.*f', 4, 7, 4, 7, 2, 3.14159) AS b, \ + printf('%q|%Q|%Q|%w', 'it''s', 'x', NULL, 'a\"b') AS c, \ + printf('%.3c|%r|%r|%r', 'ab', 1, 12, 23) AS d, \ + printf('%.20f|%!.20f|%.0f|%g|%#g', 0.1, 0.1, 2.5, 0.00001, 1.0) AS e, \ + printf('%-4s|%!-4s|', 'é', 'é') AS f, printf('') AS g, printf('%5') AS h, \ + printf('a%yb') AS i FROM one", + "a,b,c,d,e,f,g,h,i\n\"0xff|010|-1,234,567|007|18446744073709551615\",\ + \" 7|7 |3.14\",\"it''s|'x'|NULL|a\"\"\"\"b\",aaa|1st|12th|23rd,\ + 0.10000000000000000000|0.1000000000000000055|2|1e-05|1.00000,é |é |,,,a", + ) + } + + /// QUOTE gives the SQL literal of a value. + #[test] + fn quote_values() -> R { + assert_query_on( + &[ONE], + "SELECT quote('a''b') AS a, quote(12) AS b, quote(NULL) AS c, quote(1.5) AS d, \ + quote(0.1) AS e, quote(1e999) AS f, quote(1.0/3) AS g FROM one", + "a,b,c,d,e,f,g\n'a''b',12,NULL,1.5,0.1,Inf,3.333333333333333148e-01", + ) + } + /// TYPEOF sees NULL (it must not short-circuit to NULL itself). #[test] fn typeof_values() -> R { diff --git a/tests/subquery/exact.rs b/tests/subquery/exact.rs index 6e1ba2f..8e77b1c 100644 --- a/tests/subquery/exact.rs +++ b/tests/subquery/exact.rs @@ -378,6 +378,6 @@ fn in_subquery_with_two_columns_is_error() -> TestResult { fn row_value_in_subquery_is_unsupported() -> TestResult { check_fails( "SELECT id FROM emp WHERE (id, name) IN (SELECT id, name FROM emp)", - "Unsupported", + "not supported", ) } diff --git a/tests/subquery/mod.rs b/tests/subquery/mod.rs index 6fb182b..f71fd86 100644 --- a/tests/subquery/mod.rs +++ b/tests/subquery/mod.rs @@ -1,7 +1,6 @@ //! Scalar subquery tests for sqawk VM //! //! This file contains tests for scalar subqueries with aggregate functions. -//! Phase 4F: Subquery support mod exact; diff --git a/tests/transaction/mod.rs b/tests/transaction/mod.rs new file mode 100644 index 0000000..006deac --- /dev/null +++ b/tests/transaction/mod.rs @@ -0,0 +1,470 @@ +//! BEGIN / COMMIT / ROLLBACK over the in-memory tables, and how they meet +//! `--write` and the REPL; plus the writeback rules that decide which tables +//! are written (scratch tables, DROP, read-only files, hard links). +//! +//! Expected results follow SQLite 3.45. + +use std::fs; +use std::io::Write; +use std::path::Path; +use std::process::{Command, Output, Stdio}; + +use crate::helpers::create_temp_dir; + +type R = Result<(), Box>; + +const TX: &str = "k,n\n1,10\n2,20\n"; + +/// Run sqawk in `dir` with `args`, `stdin` piped in. +fn run(dir: &Path, args: &[&str], stdin: &str) -> std::io::Result { + let mut child = Command::new(assert_cmd::cargo::cargo_bin!("sqawk")) + .args(args) + .current_dir(dir) + .env("HOME", dir) + .env("XDG_STATE_HOME", dir) + .stdin(Stdio::piped()) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .spawn()?; + // A process that rejects its arguments can exit before reading stdin; + // the write then fails with BrokenPipe, which is not a test failure. + match child.stdin.take().unwrap().write_all(stdin.as_bytes()) { + Err(e) if e.kind() != std::io::ErrorKind::BrokenPipe => return Err(e), + _ => {} + } + child.wait_with_output() +} + +/// Run `-s` statements (with `extra` args) over `tx.csv` holding `TX`; +/// return (output, tx.csv contents afterwards). +fn run_tx(sqls: &[&str], extra: &[&str]) -> Result<(Output, String), Box> { + let dir = create_temp_dir()?; + fs::write(dir.path().join("tx.csv"), TX)?; + let mut args: Vec<&str> = extra.to_vec(); + for s in sqls { + args.push("-s"); + args.push(s); + } + args.push("tx.csv"); + let out = run(dir.path(), &args, "")?; + let file = fs::read_to_string(dir.path().join("tx.csv"))?; + Ok((out, file)) +} + +fn stdout(out: &Output) -> String { + String::from_utf8_lossy(&out.stdout).into_owned() +} + +fn stderr(out: &Output) -> String { + String::from_utf8_lossy(&out.stderr).into_owned() +} + +fn assert_ok(out: &Output) { + assert!(out.status.success(), "failed: {}", stderr(out)); +} + +/// The CLI error contract: exit 1, one stderr line containing `needle`. +fn assert_error(out: &Output, needle: &str) { + let err = stderr(out); + assert_eq!(out.status.code(), Some(1), "stderr: {err}"); + assert_eq!(err.lines().count(), 1, "stderr: {err}"); + assert!(err.contains(needle), "stderr: {err}"); +} + +#[test] +fn rollback_undoes_ddl() -> R { + let (out, _) = run_tx( + &[ + "BEGIN", + "CREATE TABLE q (a INT)", + "ALTER TABLE tx ADD COLUMN z TEXT", + "INSERT INTO tx VALUES (3, 30, 'x')", + "DROP TABLE tx", + "ROLLBACK", + "SELECT * FROM tx", + ], + &[], + )?; + assert_ok(&out); + assert_eq!(stdout(&out), "k,n\n1,10\n2,20\n"); + + let (out, _) = run_tx( + &["BEGIN; CREATE TABLE q (a INT); ROLLBACK; SELECT * FROM q"], + &[], + )?; + assert_error(&out, "no such table: q"); + Ok(()) +} + +#[test] +fn rollback_undoes_truncate_and_delete() -> R { + let (out, file) = run_tx( + &[ + "BEGIN; DELETE FROM tx WHERE k = 1; TRUNCATE TABLE tx; ROLLBACK", + "SELECT * FROM tx", + ], + &["--write"], + )?; + assert_ok(&out); + assert_eq!(stdout(&out), "k,n\n1,10\n2,20\n"); + assert_eq!(file, TX); + Ok(()) +} + +/// A change made before BEGIN stays, and is written, after ROLLBACK. +#[test] +fn rollback_keeps_earlier_changes() -> R { + let (out, file) = run_tx( + &[ + "UPDATE tx SET n = 5 WHERE k = 1", + "BEGIN", + "UPDATE tx SET n = 6", + "ROLLBACK", + ], + &["--write"], + )?; + assert_ok(&out); + assert_eq!(file, "k,n\n1,5\n2,20\n"); + Ok(()) +} + +/// DROP inside a rolled-back transaction: the table, and its pending +/// change from before BEGIN, come back and are written. +#[test] +fn rollback_of_drop_restores_pending_write() -> R { + let (out, file) = run_tx( + &[ + "INSERT INTO tx VALUES (3, 30)", + "BEGIN; DROP TABLE tx; ROLLBACK", + ], + &["--write"], + )?; + assert_ok(&out); + assert_eq!(file, "k,n\n1,10\n2,20\n3,30\n"); + Ok(()) +} + +#[test] +fn transactions_in_sequence() -> R { + let (out, file) = run_tx( + &[ + "BEGIN; INSERT INTO tx VALUES (3, 30); COMMIT", + "BEGIN; UPDATE tx SET n = 0; ROLLBACK", + "BEGIN IMMEDIATE TRANSACTION; DELETE FROM tx WHERE k = 1; END TRANSACTION", + "SELECT * FROM tx", + ], + &["--write"], + )?; + assert_ok(&out); + assert_eq!(stdout(&out), "k,n\n2,20\n3,30\n"); + assert_eq!(file, "k,n\n2,20\n3,30\n"); + Ok(()) +} + +#[test] +fn begin_modifiers_are_plain_begin() -> R { + for begin in [ + "BEGIN DEFERRED", + "BEGIN IMMEDIATE", + "BEGIN EXCLUSIVE TRANSACTION", + ] { + let (out, _) = run_tx( + &[begin, "UPDATE tx SET n = 0", "ROLLBACK", "SELECT n FROM tx"], + &[], + )?; + assert_ok(&out); + assert_eq!(stdout(&out), "n\n10\n20\n", "{begin}"); + } + Ok(()) +} + +/// A transaction may span `-f` files and `-s` arguments. +#[test] +fn transaction_spans_script_files() -> R { + let dir = create_temp_dir()?; + fs::write(dir.path().join("tx.csv"), TX)?; + fs::write(dir.path().join("a.sql"), "BEGIN;\nUPDATE tx SET n = 0;\n")?; + let out = run( + dir.path(), + &[ + "-f", + "a.sql", + "-s", + "ROLLBACK; SELECT n FROM tx", + "-w", + "tx.csv", + ], + "", + )?; + assert_ok(&out); + assert_eq!(stdout(&out), "n\n10\n20\n"); + assert_eq!(fs::read_to_string(dir.path().join("tx.csv"))?, TX); + Ok(()) +} + +/// Under -v the rollback of a transaction left open is reported. +#[test] +fn open_transaction_rollback_is_reported_verbose() -> R { + let (out, file) = run_tx(&["BEGIN", "UPDATE tx SET n = 0"], &["-v", "-w"])?; + assert_ok(&out); + assert!( + stderr(&out).contains("Transaction still open at exit: rolled back"), + "stderr: {}", + stderr(&out) + ); + assert_eq!(file, TX); + Ok(()) +} + +/// REPL with `.write on`: nothing is saved inside a transaction; `.save` +/// and `.load` are refused there; COMMIT saves. +#[test] +fn repl_write_mode_waits_for_commit() -> R { + let dir = create_temp_dir()?; + fs::write(dir.path().join("tx.csv"), TX)?; + fs::write(dir.path().join("other.csv"), "a\n1\n")?; + let script = ".write on\nBEGIN;\nUPDATE tx SET n = 7;\n.save\n.load other.csv\n\ + ROLLBACK;\nBEGIN;\nUPDATE tx SET n = 8 WHERE k = 2;\nCOMMIT;\n"; + let out = run(dir.path(), &["-i", "tx.csv"], script)?; + let err = stderr(&out); + assert!(err.contains("cannot save within a transaction"), "{err}"); + assert!( + err.contains("cannot load a file within a transaction"), + "{err}" + ); + assert_eq!( + fs::read_to_string(dir.path().join("tx.csv"))?, + "k,n\n1,10\n2,8\n" + ); + Ok(()) +} + +/// A REPL session that ends inside a transaction discards it, even with +/// `.write on`. +#[test] +fn repl_exit_discards_open_transaction() -> R { + let dir = create_temp_dir()?; + fs::write(dir.path().join("tx.csv"), TX)?; + let out = run( + dir.path(), + &["-i", "tx.csv"], + ".write on\nBEGIN;\nUPDATE tx SET n = 9;\n.quit\n", + )?; + assert!(out.status.success(), "{}", stderr(&out)); + assert_eq!(fs::read_to_string(dir.path().join("tx.csv"))?, TX); + Ok(()) +} + +/// A failed statement inside a REPL transaction leaves the transaction +/// open, as in SQLite; ROLLBACK then undoes the earlier change. +#[test] +fn repl_error_keeps_transaction_open() -> R { + let dir = create_temp_dir()?; + fs::write(dir.path().join("tx.csv"), TX)?; + let out = run( + dir.path(), + &["-i", "tx.csv"], + "BEGIN;\nUPDATE tx SET n = 0;\nSELECT nosuch FROM tx;\nROLLBACK;\nSELECT n FROM tx;\n", + )?; + assert!(stdout(&out).contains("n\n10\n20\n"), "{}", stdout(&out)); + assert!(stderr(&out).contains("no such column: nosuch")); + Ok(()) +} + +// --------------------------------------------------------------------------- +// Writeback rules +// --------------------------------------------------------------------------- + +/// CREATE TABLE ... AS makes a scratch table: skipped, not an error. +#[test] +fn write_skips_ctas_table() -> R { + let (out, file) = run_tx( + &[ + "CREATE TABLE s AS SELECT k FROM tx", + "INSERT INTO s VALUES (9)", + "INSERT INTO tx VALUES (3, 30)", + ], + &["-w"], + )?; + assert_ok(&out); + assert_eq!(file, "k,n\n1,10\n2,20\n3,30\n"); + Ok(()) +} + +/// A table modified and dropped in the same `-s` is not written. +#[test] +fn write_after_drop_in_one_script() -> R { + let (out, file) = run_tx(&["INSERT INTO tx VALUES (3, 30); DROP TABLE tx"], &["-w"])?; + assert_ok(&out); + assert_eq!(file, TX); + Ok(()) +} + +/// A table read from standard input still cannot be written back. +#[test] +fn write_of_stdin_table_is_error() -> R { + let dir = create_temp_dir()?; + let out = run( + dir.path(), + &["-w", "-s", "INSERT INTO stdin VALUES (2)", "-"], + "x\n1\n", + )?; + assert_error(&out, "standard input"); + Ok(()) +} + +#[cfg(unix)] +fn perms_enforced(dir: &Path) -> bool { + let p = dir.join(".perm_probe"); + fs::write(&p, "x").unwrap(); + chmod(&p, 0o000); + let enforced = fs::File::open(&p).is_err(); + fs::remove_file(&p).unwrap(); + enforced +} + +#[cfg(unix)] +fn chmod(p: &Path, mode: u32) { + let mut perm = fs::metadata(p).unwrap().permissions(); + std::os::unix::fs::PermissionsExt::set_mode(&mut perm, mode); + fs::set_permissions(p, perm).unwrap(); +} + +/// A file in a directory that cannot be written is refused before any +/// file -- including a writable one sorted first -- is written. +#[cfg(unix)] +#[test] +fn write_into_read_only_directory_is_refused_up_front() -> R { + let dir = create_temp_dir()?; + if !perms_enforced(dir.path()) { + return Ok(()); + } + fs::write(dir.path().join("a.csv"), "x\n1\n")?; + fs::create_dir(dir.path().join("rod"))?; + fs::write(dir.path().join("rod/z.csv"), "x\n1\n")?; + chmod(&dir.path().join("rod"), 0o555); + let out = run( + dir.path(), + &[ + "-w", + "-s", + "INSERT INTO a VALUES (2)", + "-s", + "INSERT INTO z VALUES (2)", + "a.csv", + "rod/z.csv", + ], + "", + ); + chmod(&dir.path().join("rod"), 0o755); + let out = out?; + assert_error(&out, "rod/z.csv"); + assert_eq!(fs::read_to_string(dir.path().join("a.csv"))?, "x\n1\n"); + assert_eq!(fs::read_to_string(dir.path().join("rod/z.csv"))?, "x\n1\n"); + Ok(()) +} + +/// A read-only file that was not modified does not block writeback. +#[cfg(unix)] +#[test] +fn unmodified_read_only_file_is_fine() -> R { + let dir = create_temp_dir()?; + fs::write(dir.path().join("rw.csv"), "x\n1\n")?; + fs::write(dir.path().join("ro.csv"), "x\n1\n")?; + chmod(&dir.path().join("ro.csv"), 0o444); + let out = run( + dir.path(), + &["-w", "-s", "INSERT INTO rw VALUES (2)", "rw.csv", "ro.csv"], + "", + ); + chmod(&dir.path().join("ro.csv"), 0o644); + assert_ok(&out?); + assert_eq!(fs::read_to_string(dir.path().join("rw.csv"))?, "x\n1\n2\n"); + Ok(()) +} + +/// Writeback replaces the file (atomic rename), so a hard link to it keeps +/// the old contents. +#[test] +fn write_splits_hard_link() -> R { + let dir = create_temp_dir()?; + fs::write(dir.path().join("tx.csv"), TX)?; + fs::hard_link(dir.path().join("tx.csv"), dir.path().join("link.csv"))?; + let out = run( + dir.path(), + &["-w", "-s", "UPDATE tx SET n = 0", "tx.csv"], + "", + )?; + assert_ok(&out); + assert_eq!( + fs::read_to_string(dir.path().join("tx.csv"))?, + "k,n\n1,0\n2,0\n" + ); + assert_eq!(fs::read_to_string(dir.path().join("link.csv"))?, TX); + Ok(()) +} + +/// REPL `.save` of a read-only file is refused too. +#[cfg(unix)] +#[test] +fn repl_save_of_read_only_file_is_refused() -> R { + let dir = create_temp_dir()?; + if !perms_enforced(dir.path()) { + return Ok(()); + } + let ro = dir.path().join("ro.csv"); + fs::write(&ro, "a\n1\n")?; + chmod(&ro, 0o444); + let out = run( + dir.path(), + &["-i", "ro.csv"], + "INSERT INTO ro VALUES (2);\n.save ro\n", + ); + chmod(&ro, 0o644); + let err = stderr(&out?); + assert!( + err.contains("cannot write 'ro.csv': file is read-only"), + "{err}" + ); + assert_eq!(fs::read_to_string(&ro)?, "a\n1\n"); + Ok(()) +} + +// --------------------------------------------------------------------------- +// --tabledef validation +// --------------------------------------------------------------------------- + +#[test] +fn tabledef_errors() -> R { + for (defs, needle) in [ + (&["--tabledef=:a,b"][..], "empty table name"), + (&["--tabledef=tx:a,A"][..], "duplicate column name: A"), + (&["--tabledef=tx:a,,b"][..], "empty column name"), + ( + &["--tabledef=tx:a,b", "--tabledef=tx:c,d"][..], + "defined twice", + ), + (&["--tabledef=zz:a"][..], "no input file loads"), + ] { + let (out, _) = run_tx(&["SELECT * FROM tx"], defs)?; + assert_error(&out, needle); + } + Ok(()) +} + +/// The REPL applies the same check at startup. +#[test] +fn tabledef_unknown_table_in_repl_is_error() -> R { + let dir = create_temp_dir()?; + fs::write(dir.path().join("tx.csv"), TX)?; + let out = run( + dir.path(), + &["-i", "--tabledef=zz:a", "tx.csv"], + "SELECT 1;\n", + )?; + assert_error( + &out, + "--tabledef names table 'zz', which no input file loads", + ); + Ok(()) +} diff --git a/tests/unsupported/mod.rs b/tests/unsupported/mod.rs index a81fe72..48c349a 100644 --- a/tests/unsupported/mod.rs +++ b/tests/unsupported/mod.rs @@ -75,10 +75,12 @@ fn delete_clauses() -> R { #[test] fn query_clauses() -> R { assert_rejected(&[ - "WITH h AS (SELECT * FROM emp) SELECT id FROM h", "SELECT id FROM emp FETCH FIRST 2 ROWS ONLY", "SELECT id FROM emp FOR UPDATE", - "SELECT id FROM emp WHERE id IN (WITH h AS (SELECT 1) SELECT * FROM h)", + "WITH h AS (SELECT * FROM emp) SELECT id FROM h FETCH FIRST 1 ROWS ONLY", + "SELECT 1 UNION (WITH h AS (SELECT 2) SELECT * FROM h)", + "WITH h AS (SELECT id FROM emp) DELETE FROM emp WHERE id IN (SELECT id FROM h) RETURNING id", + "WITH h AS (SELECT 1) UPDATE emp SET id = 2 FROM one", ]) } @@ -142,6 +144,52 @@ fn create_table_clauses() -> R { ]) } +/// CREATE TABLE options sqawk would otherwise ignore: a storage format other +/// than TEXTFILE, a WITH option other than DELIMITER / HEADER / COMMENTS, +/// HEADER or COMMENTS without a LOCATION file, other option forms, a +/// comment, and file options or a column list on CREATE TABLE ... AS (which +/// never has a file). +#[test] +fn create_table_options() -> R { + assert_rejected(&[ + "CREATE TABLE z (id INTEGER) LOCATION 'z.pq' STORED AS PARQUET", + "CREATE TABLE z (id INTEGER) STORED AS ORC", + "CREATE TABLE z (id INTEGER) STORED AS INPUTFORMAT 'a' OUTPUTFORMAT 'b'", + "CREATE TABLE z (id INTEGER) LOCATION 'z.csv' WITH (quote = '\"')", + "CREATE TABLE z (id INTEGER) WITH (header = 'false')", + "CREATE TABLE z (id INTEGER) WITH (comments = true)", + "CREATE TABLE z (id INTEGER) LOCATION 'emp.csv' WITH (header = false, x = 1)", + "CREATE TABLE z (id INTEGER) TBLPROPERTIES ('delimiter' = '|')", + "CREATE TABLE z (id INTEGER) COMMENT 'c'", + "CREATE TEMPORARY TABLE z (id INTEGER) LOCATION 'z.csv'", + "CREATE TABLE z LOCATION 'z.csv' AS SELECT * FROM emp", + "CREATE TABLE z (a INT) AS SELECT * FROM emp", + ]) +} + +/// Transaction forms with semantics sqawk does not implement. Each runs +/// after a working BEGIN and a change, which must not be written. +#[test] +fn transaction_clauses() -> R { + let mut sqls = Vec::new(); + for stmt in [ + "SAVEPOINT s", + "RELEASE s", + "RELEASE SAVEPOINT s", + "ROLLBACK TO s", + "ROLLBACK TO SAVEPOINT s", + "COMMIT AND CHAIN", + "ROLLBACK AND CHAIN", + ] { + sqls.push(format!("BEGIN; UPDATE emp SET salary = 0; {stmt}; COMMIT")); + } + sqls.push("START TRANSACTION".to_string()); + sqls.push("BEGIN TRANSACTION ISOLATION LEVEL SERIALIZABLE".to_string()); + sqls.push("BEGIN TRY".to_string()); + let refs: Vec<&str> = sqls.iter().map(String::as_str).collect(); + assert_rejected(&refs) +} + /// The red-list case: `UPDATE ... FROM` used to update every row from /// nothing; the documentation said it was "accepted without effect". #[test] diff --git a/tests/window/mod.rs b/tests/window/mod.rs index 7874551..7c2931a 100644 --- a/tests/window/mod.rs +++ b/tests/window/mod.rs @@ -472,9 +472,9 @@ fn nth_value_percent_rank_cume_dist() -> TestResult { check( "SELECT id, NTH_VALUE(id, 2) OVER (ORDER BY id) AS n, PERCENT_RANK() OVER (ORDER BY v) AS p, \ CUME_DIST() OVER (ORDER BY v) AS c FROM w", - "id,n,p,c\n1,,0.2857142857142857,0.5\n2,2,0.7142857142857143,0.875\n\ - 3,2,0.7142857142857143,0.875\n4,2,0.14285714285714285,0.25\n5,2,0,0.125\n\ - 6,2,0.5714285714285714,0.625\n7,2,1,1\n8,2,0.2857142857142857,0.5", + "id,n,p,c\n1,,0.285714285714286,0.5\n2,2,0.714285714285714,0.875\n\ + 3,2,0.714285714285714,0.875\n4,2,0.142857142857143,0.25\n5,2,0.0,0.125\n\ + 6,2,0.571428571428571,0.625\n7,2,1.0,1.0\n8,2,0.285714285714286,0.5", ) } @@ -517,14 +517,14 @@ fn window_over_join() -> TestResult { check( "SELECT w.id, cat.label, SUM(w.v) OVER (PARTITION BY cat.label) AS s \ FROM w JOIN cat ON w.g = cat.g ORDER BY w.id", - "w.id,cat.label,s\n1,Alpha,60\n2,Alpha,60\n3,Alpha,60\n4,Beta,20\n5,Beta,20\n\ + "id,label,s\n1,Alpha,60\n2,Alpha,60\n3,Alpha,60\n4,Beta,20\n5,Beta,20\n\ 6,Beta,20\n8,Alpha,60", )?; // A NULL-extended row is a row like any other. check( "SELECT cat.label, w.id, ROW_NUMBER() OVER (PARTITION BY cat.g ORDER BY w.id) AS rn \ FROM cat LEFT JOIN w ON w.g = cat.g", - "cat.label,w.id,rn\nAlpha,1,1\nAlpha,2,2\nAlpha,3,3\nAlpha,8,4\nBeta,4,1\n\ + "label,id,rn\nAlpha,1,1\nAlpha,2,2\nAlpha,3,3\nAlpha,8,4\nBeta,4,1\n\ Beta,5,2\nBeta,6,3\nGamma,,1", ) } @@ -568,7 +568,7 @@ fn window_in_expression() -> TestResult { check( "SELECT id, v * 2 + SUM(v) OVER (PARTITION BY g) AS e, \ ROUND(AVG(v) OVER (PARTITION BY g ORDER BY id), 1) AS r FROM w", - "id,e,r\n7,90,30\n1,80,10\n2,100,15\n3,100,16.7\n8,80,15\n4,30,5\n5,,5\n6,50,10", + "id,e,r\n7,90,30.0\n1,80,10.0\n2,100,15.0\n3,100,16.7\n8,80,15.0\n4,30,5.0\n5,,5.0\n6,50,10.0", ) } @@ -623,7 +623,29 @@ fn sum_over_integers_stays_integer() -> TestResult { )?; check( "SELECT AVG(v) OVER (PARTITION BY g) AS a FROM w WHERE g = 'b'", - "a\n10\n10\n10", + "a\n10.0\n10.0\n10.0", + ) +} + +/// Moving frames follow SQLite's step / inverse evaluation (values from +/// SQLite 3.45): SUM stays a real once a real has been in the frame, an +/// emptied GROUP_CONCAT is NULL, MIN / MAX and COUNT over frames that +/// start ahead of the current row. +#[test] +fn moving_frames_match_sqlite() -> TestResult { + let dir = create_temp_dir()?; + let f = dir.path().join("m.csv"); + std::fs::write(&f, "i,x,s\n1,1.5,a\n2,2,\n3,3,\n4,4,b\n5,,cc\n6,6,d\n")?; + assert_query( + &[f.to_str().unwrap()], + "SELECT i, SUM(x) OVER (ORDER BY i ROWS 1 PRECEDING) AS s, \ + GROUP_CONCAT(s, '-') OVER (ORDER BY i ROWS BETWEEN 1 PRECEDING AND CURRENT ROW) AS g, \ + MIN(x) OVER (ORDER BY i ROWS BETWEEN CURRENT ROW AND 1 FOLLOWING) AS mn, \ + COUNT(x) OVER (ORDER BY i ROWS BETWEEN 2 FOLLOWING AND UNBOUNDED FOLLOWING) AS c, \ + MAX(s) OVER (ORDER BY i ROWS BETWEEN 1 FOLLOWING AND 2 FOLLOWING) AS mx, \ + TOTAL(x) OVER (ORDER BY i ROWS BETWEEN 3 PRECEDING AND 2 PRECEDING) AS t FROM m", + "i,s,g,mn,c,mx,t\n1,1.5,a,1.5,3,,0.0\n2,3.5,a,2,2,b,0.0\n3,5.0,,3,1,cc,1.5\n\ + 4,7.0,b,4,1,d,3.5\n5,4.0,b-cc,6,0,d,5.0\n6,6.0,cc-d,6,0,,7.0", ) } diff --git a/tsq/Cargo.toml b/tsq/Cargo.toml index aabdcdd..6f52e06 100644 --- a/tsq/Cargo.toml +++ b/tsq/Cargo.toml @@ -1,9 +1,9 @@ [package] name = "tsq" -version = "0.1.0" +version = "0.2.0" edition = "2021" authors = ["Jeff Garzik "] -description = "Deterministic test data and SQL query generator for exercising sqawk" +description = "Test data generator, differential tester and fuzzer for sqawk" license = "MIT" rust-version = "1.88" # Local development tool only; never published to crates.io. @@ -11,9 +11,15 @@ publish = false [dependencies] anyhow = "1" -libc = "0.2" clap = { version = "4", features = ["derive"] } +# Portable seeded RNG: the same seed gives the same stream on every OS. +rand = { version = "0.9", default-features = false, features = ["std"] } +rand_chacha = "0.9" +# The differential oracle. The 0.31 line bundles SQLite 3.45.0, the release +# whose semantics sqawk follows (doc/sql_reference.md); later lines bundle +# newer SQLite releases whose answers differ in places sqawk does not track. +rusqlite = { version = "0.31", features = ["bundled", "backup", "hooks"] } +tempfile = "3" [dev-dependencies] assert_cmd = "2" -tempfile = "3" diff --git a/tsq/corpus/hand1.sql b/tsq/corpus/hand1.sql new file mode 100644 index 0000000..71938fd --- /dev/null +++ b/tsq/corpus/hand1.sql @@ -0,0 +1,135 @@ +-- tsq-tables: edge +-- loader type inference cross-check +SELECT aid, typeof(k), typeof(x), typeof(y), typeof(s), typeof(g), typeof(n), typeof(m) FROM a +SELECT bid, typeof(aid), typeof(k), typeof(v), typeof(w) FROM b +SELECT cid, typeof(bid), typeof(z), typeof(g) FROM c +SELECT uid, typeof(t), length(t) FROM u +SELECT did, typeof(ts), ts FROM dt +SELECT * FROM a +SELECT * FROM u +SELECT * FROM dup +SELECT * FROM e +-- precedence +SELECT 1 IS DISTINCT FROM 2 OR 1 +SELECT 0 IS NOT DISTINCT FROM 0 AND 0 +SELECT count(*) FROM a WHERE k IS NOT DISTINCT FROM 1 AND x > 0 +SELECT count(*) FROM a WHERE k IS DISTINCT FROM NULL OR x IS NULL +SELECT NULL IS NOT DISTINCT FROM NULL, 1 IS DISTINCT FROM 1.0, 'a' IS DISTINCT FROM 'A' +SELECT k FROM d UNION ALL SELECT k FROM d INTERSECT SELECT wt FROM d +SELECT k FROM d UNION SELECT wt FROM d EXCEPT SELECT 5 +SELECT k FROM d EXCEPT SELECT 1 UNION SELECT 1 +SELECT p FROM dup UNION ALL SELECT p FROM dup EXCEPT SELECT 2 +SELECT p FROM dup INTERSECT SELECT k FROM d UNION ALL SELECT 99 +SELECT 2 + 3 * 4, (2 + 3) * 4, 2 - 3 - 4, 2 * 3 % 4, 7 % 4 * 3, 100 / 10 / 5 +SELECT - - 1, 1 - -1, -(-5), +5, - 'a' +SELECT 'a' || 1 + 2, 1 + 2 || 'x', 'x' || NULL || 'y' +SELECT 1 < 2 = 1, 2 > 1 = 0, 1 = 1 = 1, NOT 1 = 2, NOT NULL IS NULL +SELECT NOT 0 AND 0, NOT (0 AND 0), 1 OR 0 AND 0, (1 OR 0) AND 0 +SELECT 5 BETWEEN 1 AND 10 AND 0, 5 NOT BETWEEN 1 AND 3 OR 0, NULL BETWEEN 1 AND 2, 1 BETWEEN NULL AND 0 +SELECT 'abc' LIKE 'A%' = 1, 'abc' LIKE 'x%' OR 1 +SELECT 1 IN (1, 2) = 1, 3 NOT IN (1, NULL), 1 IN (NULL, 1), NULL IN (1), NULL IN () +SELECT 5 & 3, 5 | 3, 1 << 4, 256 >> 2, ~5 +SELECT -9223372036854775808 >> 1, 1 << 63, 1 << 64, 1 << -1 +SELECT CASE NULL WHEN NULL THEN 1 ELSE 0 END, CASE WHEN NULL THEN 1 ELSE 2 END, CASE 1 WHEN 1.0 THEN 'eq' ELSE 'ne' END +SELECT CASE WHEN 'x' THEN 1 ELSE 0 END, CASE WHEN '1x' THEN 1 ELSE 0 END, CASE WHEN 0.1 THEN 1 ELSE 0 END, CASE WHEN '0.0' THEN 1 ELSE 0 END +SELECT CASE 'a' WHEN 'A' THEN 1 WHEN 'a' THEN 2 END, CASE 3 WHEN 1 THEN 'x' END +-- arithmetic and overflow +SELECT 9223372036854775807 + 1, -9223372036854775808 - 1, 9223372036854775807 * 2, -9223372036854775807 - 1 +SELECT -9223372036854775808 / -1, -9223372036854775808 % -1, 9223372036854775807 / -1 +SELECT -(-9223372036854775808), - -9223372036854775808, 0 - -9223372036854775808 +SELECT 9223372036854775808, -9223372036854775809, typeof(9223372036854775808), typeof(-9223372036854775808) +SELECT abs(-9223372036854775807), abs(-3), abs(-2.5), abs(NULL), abs('x'), abs('-7') +SELECT abs(-9223372036854775808) +SELECT CAST('9223372036854775808' AS INTEGER), CAST('-9223372036854775809' AS INTEGER), CAST(9.3e18 AS INTEGER), CAST(-9.3e18 AS INTEGER) +SELECT CAST(1e19 AS INTEGER), CAST('1e19' AS INTEGER), CAST(' -12 ' AS INTEGER), CAST('+5' AS INTEGER), CAST('' AS INTEGER), CAST(NULL AS INTEGER) +SELECT CAST('3.9' AS INTEGER), CAST(-3.9 AS INTEGER), CAST('0x1A' AS INTEGER), CAST('1e3' AS REAL), CAST('.5' AS REAL), CAST('5.' AS REAL), CAST('-.5e1x' AS REAL) +SELECT CAST('3.0' AS NUMERIC), CAST('3.5' AS NUMERIC), CAST('abc' AS NUMERIC), CAST('12abc' AS NUMERIC), CAST('1e2' AS NUMERIC), CAST(' 7 ' AS NUMERIC) +SELECT typeof(CAST('3.0' AS NUMERIC)), typeof(CAST('1e2' AS NUMERIC)), typeof(CAST('9223372036854775808' AS NUMERIC)), CAST('9223372036854775808' AS NUMERIC) +SELECT CAST(12 AS TEXT), typeof(CAST(12 AS TEXT)), CAST('x' AS BLOB) IS NULL, CAST(5 AS VARCHAR(2)), CAST('7' AS BIGINT) + 1 +SELECT 5 / 0, 5 % 0, 5.0 / 0, 0 / 0, 5 / 0.0, -5 % 0.0, NULL / 0 +SELECT -7 / 2, -7 % 2, 7 % -2, -7 % -2, 7.5 % 2, -7.5 % 2, 7 % 2.5, 1 / 3.0, 2.0 * 3, typeof(2.0 * 3) +SELECT '7' / '2', '7.0' / '2', '3' + '4', typeof('3' + '4'), '3.0' + 1, typeof('3.0' + 1), ' 5' + 1, '5 ' + 1 +SELECT '1e3' * 1, '0x10' + 0, '-' + 1, '.5' + 0, '5.' + 0, 'inf' + 0, '1e400' + 0, '-1e400' + 0, '1e-400' + 0 +SELECT '12abc' * 2, 'abc' + 1, '' + 1, ' ' + 1, '+3' + 0, '--3' + 0, '1.5e' + 0, '1.5e+' + 0, '1.5e+2x' + 0 +SELECT typeof('1e3' * 1), typeof('9223372036854775807' + 0), typeof('9223372036854775808' + 0), '9223372036854775808' + 0 +SELECT 0.1 + 0.2, 1e308 * 10, -1e308 * 10, 1e308 * 10 - 1e308 * 10, typeof(1e308 * 10 - 1e308 * 10) +SELECT 4.0 / 2, typeof(4.0 / 2), 1e15 + 0.5, 123456789012345678, 1.5e300 * 1.5e300 +SELECT round(-0.5), round(0.5), round(2.5), round(-2.5), round(1.005, 2), round(2.675, 2), round(1e20, 2), round(-1e-20, 5) +SELECT round(123.456, 1), round(123.456, 10), round(NULL), round('3.7'), round('x'), round(9223372036854775807), round(5, 2) +SELECT ceil(-0.5), floor(-0.5), ceil(1.2), floor(1.8), ceil(5), floor(-5), ceil('2.1'), ceiling(NULL) +SELECT sign(-3), sign(0), sign(2.5), sign(NULL), sign('-4'), sign('x'), sign('') +SELECT max(1, 2.5, '3'), min(1, 2.5, '3'), max(1, NULL), min('a', 'B'), max('a', 'B', 1), typeof(max(1, 2.5)) +SELECT 1 = 1.0, 1 < 1.5, 2 < 10, 'a' < 'B', 'abc' < 'abd', 'Z' < 'a', '' < 'a', NULL = NULL, NULL <> 1 +SELECT 9223372036854775807 = 9223372036854775806.0, 9223372036854775807 < 9223372036854775808.0, 9007199254740993 = 9007199254740992.0 +SELECT 9223372036854775807 > 9.223372036854775807e18, -9223372036854775808 = -9.223372036854775808e18 +SELECT count(*) FROM a WHERE x = 9223372036854775807 +SELECT count(*) FROM a WHERE x > 9223372036854775806 +SELECT aid, x + 1, x - 1, x * x FROM a WHERE x IS NOT NULL +SELECT aid, x / 0, y / 0, x % 7, y * 2, -x, -y FROM a +SELECT aid, n + 0, n * 1, n || '', typeof(n + 0), length(n), CAST(n AS INTEGER), CAST(n AS REAL) FROM a +SELECT aid, m + 0, typeof(m), m || '|', length(m), CAST(m AS INTEGER) FROM a +SELECT sum(x) FROM a +SELECT sum(x) FROM a WHERE x > 0 +SELECT total(x), avg(x) FROM a +SELECT sum(9223372036854775807) FROM d +SELECT total(9223372036854775807), avg(9223372036854775807) FROM d +SELECT sum(-9223372036854775808) FROM dup WHERE p = 2 +SELECT sum(x) FROM (SELECT 9223372036854775807 AS x UNION ALL SELECT 1 UNION ALL SELECT -5) +SELECT sum(x) FROM (SELECT 9223372036854775807 AS x UNION ALL SELECT -5 UNION ALL SELECT 1) +SELECT sum(n), total(n), avg(n), count(n), min(n), max(n) FROM a +SELECT sum(m), total(m), avg(m) FROM a +SELECT sum(s), total(s), avg(s) FROM a +SELECT sum(y), total(y), avg(y), sum(DISTINCT y), avg(DISTINCT y), count(DISTINCT y) FROM a +SELECT sum(x), avg(x), min(x), max(x), count(x), count(*), total(x), group_concat(x) FROM e +SELECT k, sum(x) FROM e GROUP BY k +SELECT count(*) FROM e HAVING count(*) = 0 +SELECT count(*) FROM a HAVING count(*) > 100 +SELECT count(*), sum(p) FROM dup HAVING sum(p) > 1 +-- string functions +SELECT substr('hello', 0), substr('hello', 0, 2), substr('hello', -3), substr('hello', -3, 2), substr('hello', 3, -2), substr('hello', -10, 3), substr('hello', 2, 0), substr('hello', 10) +SELECT substr('héllo日本', 2, 3), substr('日本語テキスト', -2), substr(NULL, 1), substr('abc', NULL), substr('abc', 2, NULL), substr(12345, 2, 2) +SELECT substr('hello', 1.7, 2.9), substr('hello', '2', '2'), substr('hello', -1, -1), substr('hello', 0, -1), substr('hello', 6, -3) +SELECT length('héllo'), length('日本語'), length(''), length(NULL), length(123), length(1.5), length(-0.25), length('🎉') +SELECT instr('日本語', '語'), instr('hello', ''), instr('', ''), instr(NULL, 'a'), instr('abcabc', 'c'), instr(12345, 34) +SELECT replace('aaa', 'a', 'bb'), replace('abc', '', 'x'), replace('abc', 'b', ''), replace(NULL, 'a', 'b'), replace('abc', 'b', NULL), replace(1212, 1, 3) +SELECT trim(' x '), ltrim(' x '), rtrim(' x '), trim('xxaxx', 'x'), ltrim('abcab', 'ab'), rtrim('abcab', 'ab'), trim('', 'a'), trim(NULL) +SELECT trim('日本日', '日'), rtrim('héé', 'é'), trim(12321, 1), ltrim('aaa', 'a') +SELECT upper('abc'), lower('ABC'), upper(NULL), lower(12), upper('x%y') +SELECT hex('abc'), hex(''), hex(NULL), hex(12), hex('é'), hex(-1.5), hex(1e100) +SELECT char(72, 105), char(26085, 0x672C), char(), unicode('é'), unicode(''), unicode(NULL), unicode('日本') +SELECT concat('a', NULL, 'b', 1, 2.5), concat_ws('-', 'a', NULL, 'b'), concat_ws(NULL, 'a', 'b'), concat_ws(',', NULL, NULL), concat() +SELECT 'a' || 1.5, 'a' || 2.0, 'a' || -0.0, 'a' || 1e20, 'a' || 1e-7 +SELECT quote('a''b'), quote(1), quote(NULL), quote(1.5) +SELECT length(t), substr(t, 4990), upper(substr(t, 1, 3)), instr(t, 'y'), length(t || t), length(replace(t, 'x', 'yy')) FROM u WHERE uid = 3 +SELECT uid, length(t), hex(substr(t, 1, 2)), upper(t), lower(t), trim(t), instr(t, 'e') FROM u WHERE uid <> 3 +SELECT uid, t LIKE '%e%', t LIKE '_é%', t LIKE 'h_llo', t LIKE '%\%%' ESCAPE '\', t LIKE '%\_%' ESCAPE '\', t LIKE '%\\' ESCAPE '\' FROM u WHERE uid <> 3 +SELECT group_concat(t, '|') FROM (SELECT t FROM u WHERE uid < 3 ORDER BY uid) +-- printf +SELECT printf('%5.2f|%-5d|%+d|%05d|%x|%X|%o', 3.14159, 42, 7, 42, 255, 255, 8) +SELECT printf('%e|%E|%g|%G', 12345.678, 0.000123, 100000, 1e-5) +SELECT printf('%g', 1234567), printf('%g', 123456), printf('%g', 0.0001), printf('%g', 0.00001), printf('%.3g', 1234.5), printf('%#g', 1.0) +SELECT printf('%.0f', 0.5), printf('%.0f', 1.5), printf('%.0f', 2.5), printf('%.2f', 1.005), printf('%.1f', 0.05), printf('%.10f', 0.1) +SELECT printf('%d', 3.9), printf('%d', '12abc'), printf('%d', 'x'), printf('%d', NULL), printf('%s', NULL), printf('%d', 9223372036854775807), printf('%d', -9223372036854775808) +SELECT printf('%5s|%-5s|%.2s|%10.3s|', 'abc', 'abc', 'abc', 'abcdef'), printf('%c', 'hello'), printf('%%'), printf('%s %s', 'a') +SELECT printf('%x', -1), printf('%o', -1), printf('%u', -1), printf('%i', 42), printf('% d', 42), printf('%+.2e', 12345) +SELECT printf('%5.1f%%', 45.678), printf('%-8.3f|', -1.5), printf('%08.3f', -1.5), printf('%+08d', 42), printf('%.3d', 7) +SELECT printf('%s', 1.0), printf('%s', 0.1), printf('%s', 1e20), printf('%f', 1e20), printf('%d', 1e20), printf('%.2f', 'abc'), printf('%f', '2.5x') +SELECT printf('%10d|%-10d|', -42, -42), printf('%lld', 5), printf('%z', 5), printf('%q', 'it''s'), printf('%Q', NULL), printf('%w', 'a"b') +SELECT printf('%.*f', 2, 3.14159), printf('%*d', 5, 42), printf('%!s', 'x') +SELECT printf('%s|%s', 'ß', '日本'), printf('%-6s|', 'ß'), printf('%.1s', '日本'), printf('%3s|', '日') +SELECT printf(NULL), printf(''), printf('abc'), printf('%d %d', 1), format('%d-%s', 5, 'x') +-- LIKE edge cases +SELECT 'abc' LIKE 'ABC', 'ABC' LIKE 'abc', 'é' LIKE 'É', 'É' LIKE 'é', 'aé' LIKE 'a_', 'a日b' LIKE 'a_b' +SELECT 'a' || char(10) || 'b' LIKE 'a_b', 'a' || char(10) || 'b' LIKE 'a%', '' LIKE '', '' LIKE '%', '' LIKE '_', NULL LIKE '%', 'a' LIKE NULL +SELECT 123 LIKE '1%', 1.5 LIKE '1._', 'aXb' LIKE 'a%%b', 'ab' LIKE 'a%%%b', 'abc' LIKE '%b', 'abc' LIKE '%c%', 'abc' LIKE 'a%c%' +SELECT '100%' LIKE '100\%' ESCAPE '\', '100' LIKE '100\%' ESCAPE '\', 'a_c' LIKE 'a\_c' ESCAPE '\', 'abc' LIKE 'a\_c' ESCAPE '\', 'a\c' LIKE 'a\\c' ESCAPE '\' +SELECT 'abc' LIKE 'a%' ESCAPE 'a', 'a%' LIKE 'aa%' ESCAPE 'a', 'x' LIKE 'x' ESCAPE NULL +SELECT 'abc' LIKE 'a%' ESCAPE 'ab' +SELECT 'abc' NOT LIKE 'a%', NULL NOT LIKE 'a', 'AbC' NOT LIKE '%b%' +SELECT aid, s FROM a WHERE s LIKE '%a%' +SELECT aid, s FROM a WHERE s LIKE '_' +SELECT aid, s FROM a WHERE s LIKE '%' || g || '%' +SELECT aid, s FROM a WHERE s NOT LIKE '%a%' +SELECT 'abcdefghij' LIKE '%a%b%c%d%e%f%g%h%i%j%', 'aaaaaaaaaaaaaaaaaaaaaaaaaaaaab' LIKE '%a%a%a%a%a%a%a%a%a%c' +SELECT 'x' GLOB 'x' +SELECT 'x' REGEXP 'x' diff --git a/tsq/corpus/hand2.sql b/tsq/corpus/hand2.sql new file mode 100644 index 0000000..29f8fa9 --- /dev/null +++ b/tsq/corpus/hand2.sql @@ -0,0 +1,188 @@ +-- tsq-tables: edge +-- aggregates / group by +SELECT count(DISTINCT p), count(DISTINCT q), count(p), count(*), sum(DISTINCT p), avg(DISTINCT p), group_concat(DISTINCT q) FROM dup +SELECT p, q, count(*) FROM dup GROUP BY p, q +SELECT DISTINCT p, q FROM dup +SELECT DISTINCT q FROM dup +SELECT p FROM dup UNION SELECT p FROM dup +SELECT p, q FROM dup INTERSECT SELECT p, q FROM dup +SELECT p, q FROM dup EXCEPT SELECT 1, 'a' +SELECT count(*) FROM (SELECT DISTINCT p, q FROM dup) +SELECT g, count(*), sum(x), min(x), max(x), count(DISTINCT k) FROM a GROUP BY g +SELECT g, count(*) FROM a GROUP BY 1 HAVING count(*) > 2 +SELECT k % 2 AS par, count(*), sum(y) FROM a GROUP BY par +SELECT k % 2, count(*) FROM a GROUP BY k % 2 HAVING sum(y) > 0 +SELECT upper(g) AS ug, count(*) FROM a GROUP BY ug +SELECT k, max(y), s FROM a GROUP BY k +SELECT k, min(aid), s FROM a GROUP BY k +SELECT g, max(aid), s, n FROM a GROUP BY g +SELECT max(aid), s FROM a +SELECT min(y), aid FROM a +SELECT count(*), aid FROM a WHERE k = 99 +SELECT g, avg(x) FROM a GROUP BY g HAVING avg(x) IS NULL +SELECT k, group_concat(aid) FROM (SELECT * FROM a ORDER BY aid) GROUP BY k +SELECT k, group_concat(aid, '; ') FROM (SELECT * FROM a ORDER BY aid DESC) GROUP BY k +SELECT group_concat(s, '') FROM (SELECT s FROM a ORDER BY aid) +SELECT string_agg(g, '|') FROM (SELECT g FROM a ORDER BY aid) +SELECT k, count(*) FILTER (WHERE x > 0), sum(x) FILTER (WHERE g = 'red'), count(*) FROM a GROUP BY k +SELECT count(*) FILTER (WHERE 0), sum(1) FILTER (WHERE NULL), total(1) FILTER (WHERE 0) FROM a +SELECT g, count(*) FROM a WHERE g IS NOT NULL GROUP BY g ORDER BY count(*) DESC, g +O:SELECT g, count(*) AS c FROM a GROUP BY g ORDER BY c DESC, g NULLS LAST +O:SELECT k, sum(x) AS sx FROM a GROUP BY k ORDER BY sx NULLS FIRST, k +SELECT count(*) FROM a GROUP BY NULL +SELECT 1 FROM a GROUP BY k HAVING 1 +SELECT sum(x) + count(*), max(y) - min(y), avg(k) * 2 FROM a +SELECT k, sum(x) / count(*), sum(y) / count(y) FROM a GROUP BY k +SELECT (SELECT count(*) FROM b WHERE b.aid = a.aid) AS nb, count(*) FROM a GROUP BY nb +SELECT k, count(*) FROM a GROUP BY k HAVING max(y) > (SELECT avg(z) FROM c) +SELECT min(s), max(s), min(g), max(g) FROM a +SELECT min(w), max(w) FROM b +SELECT max(t), min(t) FROM u +SELECT k, avg(y), sum(y), total(y) FROM a GROUP BY k +SELECT aid, count(*) FROM a LEFT JOIN b USING (aid) GROUP BY aid +SELECT count(*), count(b.bid), count(DISTINCT b.k) FROM a LEFT JOIN b ON a.aid = b.aid +-- window functions +O:SELECT aid, k, row_number() OVER (PARTITION BY k ORDER BY aid), rank() OVER (ORDER BY k), dense_rank() OVER (ORDER BY k) FROM a ORDER BY aid +O:SELECT aid, k, sum(x) OVER (ORDER BY k) FROM a ORDER BY aid +O:SELECT aid, sum(aid) OVER (ORDER BY aid ROWS BETWEEN 2 PRECEDING AND CURRENT ROW), avg(aid) OVER (ORDER BY aid ROWS BETWEEN 1 PRECEDING AND 1 FOLLOWING) FROM a ORDER BY aid +O:SELECT aid, sum(aid) OVER (ORDER BY aid ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING), count(*) OVER (ORDER BY aid ROWS BETWEEN CURRENT ROW AND UNBOUNDED FOLLOWING) FROM a ORDER BY aid +O:SELECT aid, sum(aid) OVER (ORDER BY aid ROWS BETWEEN 3 FOLLOWING AND 5 FOLLOWING), max(aid) OVER (ORDER BY aid ROWS BETWEEN 5 PRECEDING AND 2 PRECEDING) FROM a ORDER BY aid +O:SELECT aid, k, sum(aid) OVER (PARTITION BY k ORDER BY aid RANGE BETWEEN CURRENT ROW AND UNBOUNDED FOLLOWING) FROM a ORDER BY aid +O:SELECT aid, k, sum(aid) OVER (ORDER BY k RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW) FROM a ORDER BY aid +O:SELECT aid, lag(aid) OVER (ORDER BY aid), lead(aid, 2) OVER (ORDER BY aid), lag(aid, 3, -1) OVER (ORDER BY aid), lead(s, 1, 'none') OVER (ORDER BY aid) FROM a ORDER BY aid +O:SELECT aid, ntile(3) OVER (ORDER BY aid), ntile(7) OVER (ORDER BY aid), ntile(100) OVER (ORDER BY aid) FROM a ORDER BY aid +O:SELECT aid, k, percent_rank() OVER (ORDER BY k), cume_dist() OVER (ORDER BY k) FROM a ORDER BY aid +O:SELECT aid, k, first_value(aid) OVER (PARTITION BY k ORDER BY aid), last_value(aid) OVER (PARTITION BY k ORDER BY aid), nth_value(aid, 2) OVER (PARTITION BY k ORDER BY aid) FROM a ORDER BY aid +O:SELECT aid, last_value(aid) OVER (ORDER BY aid ROWS BETWEEN CURRENT ROW AND UNBOUNDED FOLLOWING), nth_value(aid, 3) OVER (ORDER BY aid ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING) FROM a ORDER BY aid +O:SELECT aid, k, min(y) OVER (PARTITION BY k), max(y) OVER (PARTITION BY k), count(y) OVER (PARTITION BY k), avg(y) OVER (PARTITION BY k) FROM a ORDER BY aid +O:SELECT aid, x, rank() OVER (ORDER BY x), rank() OVER (ORDER BY x DESC), rank() OVER (ORDER BY x NULLS LAST), dense_rank() OVER (ORDER BY x DESC NULLS FIRST) FROM a ORDER BY aid +O:SELECT k, count(*), rank() OVER (ORDER BY count(*) DESC), sum(count(*)) OVER (), sum(sum(x)) OVER (ORDER BY k) FROM a GROUP BY k ORDER BY k +O:SELECT g, sum(y), row_number() OVER (ORDER BY sum(y) DESC) FROM a GROUP BY g HAVING count(*) > 1 ORDER BY 3 +O:SELECT aid, row_number() OVER w, rank() OVER (w ORDER BY aid DESC) FROM a WINDOW w AS (PARTITION BY k) ORDER BY aid +O:SELECT aid FROM a ORDER BY row_number() OVER (ORDER BY y DESC, aid) +O:SELECT aid, round(avg(y) OVER (ORDER BY aid), 3), sum(y) OVER () - y FROM a ORDER BY aid +SELECT eid, row_number() OVER (ORDER BY eid), sum(x) OVER () FROM e +SELECT count(*) OVER () FROM a WHERE 0 +O:SELECT a.aid, b.bid, row_number() OVER (PARTITION BY a.aid ORDER BY b.bid) FROM a JOIN b ON a.aid = b.aid ORDER BY 1, 2 +SELECT * FROM (SELECT aid, rank() OVER (ORDER BY k) AS r FROM a) WHERE r = 1 +SELECT k, max(r) FROM (SELECT k, row_number() OVER (PARTITION BY k ORDER BY aid) AS r FROM a) GROUP BY k +O:SELECT p, q, row_number() OVER (ORDER BY p, q), rank() OVER (ORDER BY p, q), dense_rank() OVER (ORDER BY p, q) FROM dup ORDER BY 4, 1, 2 +O:SELECT aid, sum(x) OVER (ORDER BY aid) FROM a ORDER BY aid +SELECT sum(x) OVER (ORDER BY aid ROWS BETWEEN 1 PRECEDING AND 1 FOLLOWING) FROM a +O:SELECT aid, count(DISTINCT k) OVER (ORDER BY aid) FROM a ORDER BY aid +O:SELECT aid, lag(aid, 0) OVER (ORDER BY aid), lag(aid, -1) OVER (ORDER BY aid), nth_value(aid, 1) OVER (ORDER BY aid) FROM a ORDER BY aid +O:SELECT aid, ntile(0) OVER (ORDER BY aid) FROM a ORDER BY aid +O:SELECT aid, lag(aid, NULL) OVER (ORDER BY aid) FROM a ORDER BY aid +-- joins +SELECT * FROM a JOIN b USING (aid) +SELECT * FROM a NATURAL JOIN b +SELECT * FROM a NATURAL LEFT JOIN b +SELECT * FROM b NATURAL JOIN c +SELECT * FROM a JOIN b USING (aid) JOIN c USING (bid) +SELECT * FROM a JOIN b USING (aid, k) +SELECT * FROM a LEFT JOIN b USING (aid) LEFT JOIN c USING (bid) +SELECT * FROM d RIGHT JOIN a ON a.k = d.k +SELECT * FROM d FULL JOIN a ON a.k = d.k +SELECT * FROM d FULL JOIN a USING (k) +SELECT k, label, aid FROM d FULL JOIN a USING (k) +SELECT * FROM d NATURAL FULL JOIN dup +SELECT * FROM d NATURAL RIGHT JOIN a +SELECT d.k, a.k, k FROM d RIGHT JOIN a USING (k) +SELECT k FROM d LEFT JOIN a USING (k) +SELECT * FROM a, d WHERE a.k = d.k AND d.wt > 10 +SELECT * FROM a CROSS JOIN e +SELECT * FROM e LEFT JOIN a ON 1 +SELECT * FROM a LEFT JOIN e ON 1 +SELECT * FROM a FULL JOIN e ON a.k = e.k +SELECT * FROM e FULL JOIN e AS e2 ON 1 +SELECT a.aid, d.label FROM a LEFT JOIN d ON a.k = d.k AND d.wt > 15 +SELECT a.aid, d.label FROM a LEFT JOIN d ON a.k = d.k WHERE d.wt > 15 +SELECT a.aid FROM a LEFT JOIN d ON a.k = d.k WHERE d.k IS NULL +SELECT a.aid, d.label FROM a LEFT JOIN d ON 0 +SELECT x1.aid, x2.aid FROM a x1 JOIN a x2 ON x1.k = x2.k AND x1.aid < x2.aid +SELECT a.aid, s.kr FROM a JOIN (SELECT k * 1.0 AS kr FROM d) s ON a.k = s.kr +SELECT a.aid, s.kr FROM a JOIN (SELECT k * 1.0 AS kr FROM d) s ON a.k = s.kr OR 0 +SELECT a.aid, s.kt FROM a JOIN (SELECT CAST(k AS TEXT) AS kt FROM d) s ON a.k = s.kt +SELECT a.aid, s.kt FROM a JOIN (SELECT CAST(k AS TEXT) AS kt FROM d) s ON a.k = s.kt OR 0 +SELECT a.aid, d.k FROM a JOIN d ON a.n = d.k +SELECT a.aid, d.k FROM a JOIN d ON a.n = d.k OR 0 +SELECT a.aid, d.k FROM a, d WHERE a.n = d.k +SELECT a.aid, b.bid FROM a JOIN b ON a.aid = b.aid JOIN c ON b.bid = c.bid JOIN d ON c.k = d.k +SELECT a.aid, b.bid, c.cid, d.k FROM a LEFT JOIN b ON a.aid = b.aid LEFT JOIN c ON b.bid = c.bid LEFT JOIN d ON c.k = d.k +SELECT a.aid, b.bid, c.cid FROM a JOIN b ON a.aid = b.aid RIGHT JOIN c ON b.bid = c.bid +SELECT a.aid, b.bid, c.cid FROM a LEFT JOIN b ON a.aid = b.aid CROSS JOIN c WHERE c.cid < 203 +SELECT a.aid, b.bid FROM a JOIN b ON b.aid = a.aid AND b.k = a.k +SELECT a.aid, b.bid FROM a JOIN b ON a.aid = b.aid + 0 +SELECT count(*) FROM a JOIN b ON a.y = b.v +SELECT count(*) FROM a JOIN c ON a.y = c.z +SELECT count(*) FROM a JOIN a AS a2 ON a.x = a2.x +SELECT count(*) FROM a JOIN a AS a2 ON a.s = a2.s +SELECT count(*) FROM a JOIN a AS a2 ON a.m = a2.m +SELECT count(*) FROM a JOIN a AS a2 ON a.m = a2.m OR 0 +SELECT count(*) FROM a JOIN a AS a2 ON a.n = a2.n +SELECT count(*) FROM a JOIN a AS a2 ON a.n = a2.n OR 0 +SELECT count(*) FROM dup x JOIN dup y ON x.p = y.p +SELECT count(*) FROM dup x JOIN dup y ON x.q = y.q +SELECT count(*) FROM dup x JOIN dup y ON x.q IS y.q +SELECT * FROM a JOIN b ON c.k = a.k +SELECT * FROM a JOIN b USING (nosuch) +SELECT aid FROM a JOIN b USING (aid) WHERE b.aid > 15 +SELECT k FROM a JOIN d USING (k) +SELECT k FROM a JOIN d ON a.k = d.k +-- subqueries +SELECT aid, (SELECT count(*) FROM b WHERE b.aid = a.aid AND EXISTS (SELECT 1 FROM c WHERE c.bid = b.bid AND c.k = a.k)) FROM a +SELECT aid FROM a WHERE EXISTS (SELECT 1 FROM b WHERE b.aid = a.aid AND b.v > (SELECT min(z) FROM c WHERE c.bid = b.bid OR c.k = a.k)) +SELECT aid, (SELECT max(v) FROM b WHERE b.k = a.k AND b.bid IN (SELECT bid FROM c WHERE c.k = a.k OR c.g = a.g)) FROM a +SELECT aid, (SELECT (SELECT count(*) FROM c WHERE c.k = a.k) FROM d LIMIT 1) FROM a +SELECT aid, (SELECT label FROM d WHERE d.k = a.k ORDER BY label LIMIT 1) FROM a +SELECT aid, (SELECT label FROM d WHERE d.k = a.k ORDER BY label DESC LIMIT 1 OFFSET 1) FROM a +SELECT aid FROM a WHERE k IN (SELECT k FROM d) +SELECT aid FROM a WHERE k NOT IN (SELECT k FROM d) +SELECT aid FROM a WHERE k NOT IN (SELECT k FROM d WHERE k IS NOT NULL) +SELECT aid FROM a WHERE k IN (SELECT k FROM e) +SELECT aid FROM a WHERE k NOT IN (SELECT k FROM e) +SELECT aid, k IN (SELECT k FROM d), k NOT IN (SELECT k FROM d), NULL IN (SELECT k FROM e), NULL NOT IN (SELECT k FROM e) FROM a +SELECT aid FROM a WHERE (SELECT count(*) FROM b WHERE b.aid = a.aid) > 1 +SELECT aid FROM a WHERE NOT EXISTS (SELECT 1 FROM b WHERE b.aid = a.aid) +SELECT aid FROM a WHERE EXISTS (SELECT 1 FROM b LIMIT 0) +SELECT aid FROM a WHERE EXISTS (SELECT NULL) +SELECT (SELECT k FROM d), (SELECT k FROM d WHERE 0), (SELECT max(k) FROM e) +SELECT (SELECT k, label FROM d) +SELECT aid FROM a WHERE aid IN (SELECT aid, k FROM a) +SELECT aid FROM a ORDER BY (SELECT count(*) FROM b WHERE b.aid = a.aid) DESC, aid LIMIT 5 +SELECT aid FROM a LIMIT (SELECT count(*) FROM d) +SELECT aid FROM a LIMIT (SELECT 2) OFFSET (SELECT 18) +SELECT * FROM (SELECT * FROM (SELECT aid, k FROM a WHERE k > 1) WHERE k < 4) t WHERE aid > 5 +SELECT t.k, t.c FROM (SELECT k, count(*) AS c FROM a GROUP BY k) t JOIN d ON t.k = d.k +SELECT * FROM (SELECT aid FROM a ORDER BY aid DESC LIMIT 3) ORDER BY 1 +SELECT count(*) FROM (SELECT * FROM a UNION ALL SELECT * FROM a) +SELECT aid, (SELECT sum(x2.y) FROM (SELECT y FROM a x3 WHERE x3.k = a.k) x2) FROM a +SELECT aid FROM a WHERE x > ALL (SELECT v FROM b) +SELECT aid FROM a WHERE (k, x) IN (SELECT k, v FROM b) +SELECT aid, (SELECT count(*) FROM b WHERE b.aid = a.aid) + (SELECT count(*) FROM c WHERE c.k = a.k) FROM a +SELECT aid FROM a WHERE k = (SELECT k FROM d ORDER BY wt DESC LIMIT 1) +SELECT aid, CASE WHEN EXISTS (SELECT 1 FROM b WHERE b.aid = a.aid) THEN 'y' ELSE 'n' END FROM a +SELECT aid FROM a WHERE aid IN (SELECT aid FROM b UNION SELECT k FROM d) +SELECT aid FROM a WHERE aid IN (SELECT b.aid FROM b WHERE b.k = a.k INTERSECT SELECT aid FROM a WHERE x > 0) +SELECT count(*), (SELECT count(*) FROM b WHERE b.k = a.k) FROM a +SELECT k, (SELECT count(*) FROM b WHERE b.k = a.k) FROM a GROUP BY k +SELECT k, (SELECT sum(a.x) FROM d WHERE d.k = a.k) FROM a GROUP BY k +SELECT aid FROM a WHERE x IN (SELECT max(v) FROM b GROUP BY k) +-- compounds with order/limit +O:SELECT k FROM a UNION SELECT k FROM d ORDER BY 1 DESC LIMIT 3 +O:SELECT k, s FROM a UNION ALL SELECT k, label FROM d ORDER BY 1, 2 LIMIT 5 OFFSET 2 +O:SELECT aid AS z FROM a EXCEPT SELECT bid - 100 FROM b ORDER BY z +O:SELECT x FROM a INTERSECT SELECT v FROM b ORDER BY 1 +SELECT p, q FROM dup UNION SELECT p, q FROM dup +SELECT NULL UNION SELECT NULL +SELECT 1, 'a' UNION SELECT 1.0, 'a' +SELECT 1 UNION SELECT '1' +SELECT 1 INTERSECT SELECT 1.0 +SELECT k FROM e UNION ALL SELECT k FROM e +SELECT k FROM d EXCEPT SELECT k FROM e +SELECT k FROM e INTERSECT SELECT k FROM d +O:SELECT k FROM d UNION SELECT NULL ORDER BY 1 NULLS LAST +SELECT k FROM d UNION SELECT k, k FROM d +O:SELECT k FROM d UNION ALL SELECT wt FROM d ORDER BY k +O:SELECT k AS kk FROM d UNION SELECT wt FROM d ORDER BY kk DESC +SELECT k FROM d ORDER BY 1 UNION SELECT 1 diff --git a/tsq/corpus/hand3.sql b/tsq/corpus/hand3.sql new file mode 100644 index 0000000..3a6d7ca --- /dev/null +++ b/tsq/corpus/hand3.sql @@ -0,0 +1,118 @@ +-- tsq-tables: edge +-- DML sequences +INSERT INTO d SELECT a.k, b.w, b.v FROM a JOIN b ON a.aid = b.aid WHERE b.v > 0; SELECT * FROM d +INSERT INTO d (label) VALUES ('only'); SELECT * FROM d +INSERT INTO d (wt, k) VALUES (1, 2), (3, 4), (NULL, NULL); SELECT * FROM d +INSERT INTO d VALUES (9, 'nine', 90); INSERT INTO d SELECT k + 100, label || '!', wt FROM d; SELECT * FROM d +INSERT INTO d SELECT * FROM d; INSERT INTO d SELECT * FROM d; SELECT count(*), sum(wt) FROM d +INSERT INTO d SELECT k, label, wt FROM d ORDER BY wt DESC LIMIT 2; SELECT * FROM d +INSERT INTO d SELECT k, max(label), sum(wt) FROM d GROUP BY k; SELECT * FROM d +INSERT INTO d SELECT 1, 2; SELECT * FROM d +INSERT INTO d VALUES ((SELECT max(aid) FROM a), (SELECT min(s) FROM a), (SELECT count(*) FROM d)); SELECT * FROM d +INSERT INTO e SELECT aid, k, x FROM a WHERE x > 0; SELECT * FROM e +INSERT INTO e SELECT aid, k, x FROM a WHERE x > 0; DELETE FROM e WHERE k IN (SELECT k FROM d WHERE wt > 15); SELECT * FROM e +UPDATE d SET wt = (SELECT max(v) FROM b WHERE b.k = d.k); SELECT * FROM d +UPDATE d SET wt = (SELECT count(*) FROM a WHERE a.k = d.k), label = upper(label) WHERE k IS NOT NULL; SELECT * FROM d +UPDATE d SET k = wt, wt = k; SELECT * FROM d +UPDATE d SET wt = wt + 1, wt = wt + 2; SELECT * FROM d +UPDATE d SET wt = NULL WHERE wt > 100; SELECT * FROM d +UPDATE d SET label = label || label WHERE k IN (SELECT k FROM a WHERE x > 0); SELECT * FROM d +UPDATE d SET wt = wt * 2 WHERE EXISTS (SELECT 1 FROM b WHERE b.k = d.k AND b.v > 5); SELECT * FROM d +UPDATE d SET wt = 9223372036854775807 + wt; SELECT * FROM d +UPDATE d SET wt = (SELECT max(wt) FROM d); SELECT * FROM d +UPDATE d SET wt = (SELECT sum(x.wt) FROM d x WHERE x.k = d.k); SELECT * FROM d +UPDATE d SET nosuch = 1; SELECT * FROM d +DELETE FROM d WHERE k IN (SELECT k FROM a WHERE g = 'red'); SELECT * FROM d +DELETE FROM d WHERE k NOT IN (SELECT k FROM a); SELECT * FROM d +DELETE FROM d WHERE k NOT IN (SELECT k FROM a WHERE k IS NOT NULL); SELECT * FROM d +DELETE FROM d WHERE wt > (SELECT avg(wt) FROM d); SELECT * FROM d +DELETE FROM d WHERE wt = (SELECT max(x.wt) FROM d x WHERE x.k = d.k); SELECT * FROM d +DELETE FROM d; SELECT count(*), sum(wt) FROM d +DELETE FROM d; INSERT INTO d VALUES (1, 'x', 2); SELECT * FROM d +DELETE FROM dup WHERE p = 1; SELECT * FROM dup +DELETE FROM dup WHERE q IS NULL; INSERT INTO dup SELECT p, q FROM dup; SELECT * FROM dup +DELETE FROM a WHERE aid IN (SELECT aid FROM b) OR x IS NULL; SELECT aid FROM a +UPDATE a SET s = NULL WHERE s LIKE '%a%'; SELECT aid, s FROM a +UPDATE a SET y = y / 0, x = -x; SELECT aid, x, y FROM a +-- CTAS / ALTER +CREATE TABLE t9 AS SELECT k, count(*) AS c, sum(x) AS sx FROM a GROUP BY k; SELECT * FROM t9 +CREATE TABLE t9 AS SELECT a.aid, b.bid, a.s FROM a JOIN b ON a.aid = b.aid; SELECT * FROM t9 +CREATE TABLE t9 AS SELECT * FROM e; INSERT INTO t9 VALUES (1, 2, 3); SELECT * FROM t9 +CREATE TABLE t9 AS SELECT aid, x * 1.5 AS v FROM a; SELECT aid, v, typeof(v) FROM t9 +CREATE TABLE t9 AS SELECT 1 AS one, 'x' AS ex, NULL AS nn; SELECT * FROM t9 +CREATE TABLE t9 AS SELECT * FROM a WHERE 0; SELECT count(*) FROM t9 +CREATE TABLE t9 AS SELECT k, k FROM d; SELECT * FROM t9 +CREATE TABLE t9 AS SELECT aid, row_number() OVER (ORDER BY aid DESC) AS rn FROM a; SELECT * FROM t9 +CREATE TABLE t9 AS SELECT * FROM d; UPDATE t9 SET wt = 0; SELECT * FROM d +CREATE TABLE t9 (a INTEGER, b TEXT); INSERT INTO t9 VALUES (1, 'x'), (2, NULL); SELECT * FROM t9 +CREATE TABLE t9 (a INTEGER, b TEXT); INSERT INTO t9 VALUES ('10', 5); SELECT a, typeof(a), b, typeof(b) FROM t9 +CREATE TABLE t9 (a INTEGER, b TEXT); INSERT INTO t9 SELECT k, label FROM d; SELECT * FROM t9 WHERE a > 1 +CREATE TABLE d (z INT); SELECT * FROM d +ALTER TABLE d ADD COLUMN extra INTEGER; SELECT * FROM d +ALTER TABLE d ADD COLUMN extra TEXT; UPDATE d SET extra = label || k; SELECT * FROM d +ALTER TABLE d ADD COLUMN extra INT; INSERT INTO d VALUES (7, 'seven', 70, 700); SELECT * FROM d +ALTER TABLE d ADD COLUMN extra INT; INSERT INTO d (k) VALUES (8); SELECT * FROM d +ALTER TABLE d ADD COLUMN label TEXT; SELECT * FROM d +ALTER TABLE e ADD COLUMN y REAL; INSERT INTO e VALUES (1, 2, 3, 4.5); SELECT * FROM e +DROP TABLE d; SELECT * FROM d +DROP TABLE IF EXISTS nosuch; SELECT 1 +-- NULL heavy +SELECT NULL + 1, NULL || 'a', NULL = NULL, NULL AND 0, NULL OR 1, NULL AND 1, NULL OR 0, NOT NULL +SELECT coalesce(NULL, NULL), coalesce(NULL, 1, 2), ifnull(NULL, 'x'), nullif(1, 1), nullif(1, 2), nullif(NULL, 1), iif(NULL, 1, 2) +SELECT coalesce(x, y, k, 0), ifnull(s, ''), nullif(g, 'red'), iif(x > 0, 'pos', iif(x < 0, 'neg', iif(x = 0, 'zero', 'null'))) FROM a +SELECT CASE WHEN x IS NULL THEN coalesce(y, -1) WHEN x > 0 THEN CASE WHEN k > 2 THEN x ELSE -x END ELSE nullif(x, 0) END FROM a +SELECT coalesce(nullif(coalesce(s, g), ''), iif(n IS NULL, 'nn', n)) FROM a +SELECT count(*), count(x), count(y), count(s), count(g), count(n), count(m) FROM a +SELECT x, count(*) FROM a GROUP BY x +O:SELECT aid, x FROM a ORDER BY x, aid +O:SELECT aid, x FROM a ORDER BY x DESC, aid +O:SELECT aid, x FROM a ORDER BY x NULLS LAST, aid +O:SELECT aid, x FROM a ORDER BY x DESC NULLS FIRST, aid +O:SELECT aid, s FROM a ORDER BY s, aid +O:SELECT aid, y FROM a ORDER BY y DESC, aid LIMIT 5 +O:SELECT aid, m FROM a ORDER BY m, aid +O:SELECT aid, n FROM a ORDER BY n, aid +O:SELECT DISTINCT g FROM a ORDER BY g +O:SELECT aid FROM a ORDER BY aid LIMIT -1 +O:SELECT aid FROM a ORDER BY aid LIMIT 3 OFFSET -2 +O:SELECT aid FROM a ORDER BY aid LIMIT 0 +O:SELECT aid FROM a ORDER BY aid LIMIT '3' +O:SELECT aid FROM a ORDER BY aid LIMIT 2.5 +O:SELECT aid FROM a ORDER BY aid LIMIT 2, 3 +O:SELECT aid, k FROM a ORDER BY k COLLATE NOCASE, aid +O:SELECT s FROM a WHERE s IS NOT NULL ORDER BY lower(s), s +O:SELECT aid FROM a ORDER BY -aid LIMIT 3 +O:SELECT aid AS x FROM a ORDER BY x LIMIT 3 +O:SELECT aid, x AS aid FROM a ORDER BY aid LIMIT 3 +O:SELECT k, aid FROM a ORDER BY 1 DESC, 2 DESC +SELECT aid FROM a ORDER BY 9 +SELECT max(x) FROM a WHERE x < 9223372036854775807 +SELECT min(m), max(m) FROM a +SELECT DISTINCT m FROM a +SELECT DISTINCT n FROM a +SELECT n, count(*) FROM a GROUP BY n +SELECT m, count(*) FROM a GROUP BY m +SELECT DISTINCT y * 0 FROM a +SELECT DISTINCT k, k * 1.0 FROM a +SELECT k * 1.0 FROM a UNION SELECT k FROM a +-- date / time +SELECT ts, mod, date(ts), time(ts), datetime(ts), date(ts, mod), datetime(ts, mod), julianday(ts), unixepoch(ts) FROM dt +SELECT ts, strftime('%Y-%m-%d %H:%M:%S %j %w %W %s', ts), strftime('%f %J', ts) FROM dt +SELECT date('2024-02-29', '+1 year'), date('2024-03-31', '-1 month'), date('2024-01-31', '+1 month'), date('2023-01-31', '+1 month', 'start of month') +SELECT date('2024-13-01'), date('2024-02-30'), date('2024-02-31'), date('2024-02-32'), date('2024-00-10'), date('2023-02-29'), time('24:00'), time('23:60') +SELECT datetime('2024-01-01 10:00:00+05:30'), datetime('2024-01-01T10:00Z'), datetime('2024-01-01 10:00:00-01:15'), datetime('2024-01-01 10:00:00.123456') +SELECT datetime(1700000000, 'unixepoch'), datetime(1700000000.5, 'unixepoch'), datetime(-1, 'unixepoch'), date(2460000.5), datetime(0), julianday('2000-01-01 12:00') +SELECT strftime('%s', '1969-12-31'), strftime('%s', '2038-01-19 03:14:08'), unixepoch('1970-01-01 00:00:01.9'), strftime('%f', '2024-01-01 00:00:01.999') +SELECT date('2024-06-15', 'weekday 0'), date('2024-06-16', 'weekday 0'), date('2024-06-15', 'weekday 6'), date('2024-06-15', 'weekday 7'), date('2024-06-15', 'weekday -1') +SELECT date('2024-01-01', '+1.5 days'), datetime('2024-01-01', '+1.5 months'), datetime('2024-01-01', '-0.5 years'), datetime('2024-01-01', '+25 hours'), datetime('2024-01-01', '-90 minutes'), datetime('2024-01-01', '+3661 seconds') +SELECT date('2024-06-15', 'start of year', '-1 day'), date('2024-06-15', 'start of month', '+1 month', '-1 day'), datetime('2024-06-15 13:14:15', 'start of day') +SELECT date('2024-01-01', '+1 days', '+1 days', '+1 months', '-1 years'), date('2024-01-01', '+0001-02-03'), datetime('2024-01-01', '+01:30'), date('2024-01-01', '+1 fortnight') +SELECT date('0000-01-01', '-1 day'), date('9999-12-31', '+1 day'), date('-0001-01-01'), date('10000-01-01'), datetime('2024-01-01 25:00'), date(''), date(NULL), date('now', 'xyz') +SELECT date('2024-01-01', 'unixepoch'), datetime(1e10, 'unixepoch'), datetime(253402300800, 'unixepoch'), datetime(5373484.5), datetime(5373485), datetime(-1) +SELECT strftime('%Y', '2024-01-01', '+1 year'), strftime('%H:%M', '12:34:56'), strftime('%%|%d', '2024-05-06'), strftime('%Q', '2024-01-01'), strftime(NULL, '2024-01-01'), strftime('%Y', NULL) +SELECT strftime('%W', '2024-01-01'), strftime('%W', '2023-01-01'), strftime('%W', '2024-12-31'), strftime('%j', '2024-12-31'), strftime('%w', '2024-12-31'), strftime('%u', '2024-12-31') +SELECT julianday('2024-01-01') - julianday('2023-01-01'), julianday('-4713-11-24 12:00'), julianday('2024-01-01', 'utc'), datetime('2024-01-01', 'localtime') +SELECT time('12:30:45.678'), time('12:30'), time('2024-01-01 12:30'), datetime('12:30'), date('12:30'), time(0.5), time('T12:30') +SELECT date(20240101), date('20240101'), date('2024-1-1'), date(' 2024-01-01'), date('2024-01-01 '), datetime('2024-01-01 10:00'), date('2024/01/01') +SELECT datetime('2024-03-10 12:00', '+0.000001 days'), unixepoch('2024-01-01', 'subsec'), datetime('2024-01-01', 'auto'), date(2460000.5, 'julianday'), datetime(1700000000, 'auto') +SELECT date(ts, '+1 day', 'start of month') FROM dt WHERE did < 6 diff --git a/tests/sqlite_diff/queries.sql b/tsq/corpus/queries.sql similarity index 100% rename from tests/sqlite_diff/queries.sql rename to tsq/corpus/queries.sql diff --git a/tests/sqlite_diff/queries2.sql b/tsq/corpus/queries2.sql similarity index 83% rename from tests/sqlite_diff/queries2.sql rename to tsq/corpus/queries2.sql index fa506b0..7d960fd 100644 --- a/tests/sqlite_diff/queries2.sql +++ b/tsq/corpus/queries2.sql @@ -164,3 +164,21 @@ DELETE FROM t2 WHERE k IS NULL; SELECT t1.k FROM t1 WHERE k NOT IN (SELECT k FRO UPDATE t1 SET s = s || '!' WHERE s IS NOT NULL; SELECT * FROM t1 UPDATE t1 SET a = -a WHERE a < 0; SELECT * FROM t1 UPDATE t1 SET a = a / 4; SELECT * FROM t1 +-- common table expressions +WITH b AS (SELECT id FROM a WHERE salary > 70), a AS (SELECT id, salary FROM emp) SELECT id FROM b +WITH emp AS (SELECT * FROM main.emp WHERE salary < 80) SELECT COUNT(*), SUM(salary) FROM emp +WITH c AS (SELECT 1 AS v) SELECT (WITH c AS (SELECT 2 AS v) SELECT v FROM c), v FROM c +WITH m AS (SELECT dept, MAX(salary) AS mx FROM emp GROUP BY dept) SELECT name FROM emp e WHERE salary = (SELECT mx FROM m WHERE m.dept = e.dept) +SELECT (SELECT COUNT(*) FROM emp), (WITH emp AS (SELECT 1 AS id) SELECT COUNT(*) FROM emp) +WITH u AS (SELECT k FROM t1 UNION ALL SELECT k FROM t2 EXCEPT SELECT k FROM t3) SELECT k FROM u +WITH r AS (SELECT id FROM emp WHERE salary < 70) DELETE FROM emp WHERE id IN (SELECT id FROM r); SELECT id FROM emp +INSERT INTO t3 WITH r AS (SELECT k + 50, 'w' FROM t2 WHERE k IS NOT NULL) SELECT * FROM r; SELECT k, c FROM t3 +CREATE TABLE z AS WITH r AS (SELECT id, salary FROM emp WHERE salary > 70) SELECT * FROM r; SELECT id, salary FROM z +O:WITH RECURSIVE c(n) AS (SELECT 1 UNION ALL SELECT n + 2 FROM c WHERE n < 9) SELECT n FROM c +O:WITH RECURSIVE c(n) AS (SELECT 1 UNION ALL SELECT n + 1 FROM c LIMIT 4 OFFSET 2) SELECT n FROM c +O:WITH RECURSIVE c(n) AS (SELECT 3 UNION SELECT (n * 3) % 11 FROM c) SELECT n FROM c +O:WITH RECURSIVE c(n) AS (SELECT 1 UNION ALL SELECT n + 1 FROM c WHERE n < 3 UNION ALL SELECT n * 10 FROM c WHERE n < 3) SELECT n FROM c +O:WITH RECURSIVE sub(id, lvl) AS (SELECT id, 0 FROM emp WHERE id = 1 UNION ALL SELECT emp.id, sub.lvl + 1 FROM emp JOIN sub ON emp.mgr = sub.id) SELECT id, lvl FROM sub +O:WITH RECURSIVE sub(id, lvl) AS (SELECT id, 0 FROM emp WHERE id = 1 UNION ALL SELECT emp.id, sub.lvl + 1 FROM emp, sub WHERE emp.mgr = sub.id ORDER BY 2 DESC) SELECT id, lvl FROM sub +O:WITH RECURSIVE anc(id, mgr) AS (SELECT id, mgr FROM emp WHERE id = 9 UNION ALL SELECT emp.id, emp.mgr FROM emp, anc WHERE emp.id = anc.mgr) SELECT id, mgr FROM anc +WITH RECURSIVE sub(id) AS (SELECT 3 UNION ALL SELECT emp.id FROM emp JOIN sub ON emp.mgr = sub.id) UPDATE emp SET bonus = -1 WHERE id IN (SELECT id FROM sub); SELECT id, bonus FROM emp diff --git a/tsq/src/clifuzz.rs b/tsq/src/clifuzz.rs new file mode 100644 index 0000000..917ffec --- /dev/null +++ b/tsq/src/clifuzz.rs @@ -0,0 +1,1383 @@ +//! `tsq cli`: the command-line and REPL surface fuzzer. It absorbs the +//! retired Python tools csvfuzz.py (malformed and huge inputs) and fuzz.py +//! (garbage SQL, limit values, deep nesting) as randomized generators. +//! +//! Each iteration builds one sqawk run from a random scenario: +//! +//! * `flags`: random flag and operand combinations -- `-s` / `-f FILE` / +//! `-f -` in any order, `-F` and `--output-delimiter` (valid and +//! invalid), `--no-header`, `--no-output-header`, `--comments`, +//! `--tabledef` (good and bad), `name=path` operands (including a `k=v/` +//! directory and `=x`), `-` for standard input, missing files, +//! directories, a read-only file under `-w`, `-i` with `-s`, unknown +//! flags, `--help` / `--version`; +//! * `repl`: `-i` with a script on standard input -- dot-commands with good +//! and bad arguments (`.load`, `.save`, `.cd`, `.exit N`, ...), statements +//! split over lines, transactions, garbage, an unterminated last +//! statement; +//! * `malformed`: csvfuzz.py's malformed files (unclosed quotes, NUL bytes, +//! invalid UTF-8, mixed line endings, duplicate headers, ...) and random +//! byte-level mutations of a valid file, read and written back; +//! * `garbage`: fuzz.py's random bytes and token soup as SQL; +//! * `limits`: fuzz.py's scalar calls at the limits (printf widths of 2^31, +//! i64 extremes in substr / lag / ntile / LIMIT / round, dates past 9999, +//! ...), their numbers re-drawn from the extremes; +//! * `huge`: very large statements through `-f` (deep nesting, long +//! operator chains, big IN lists and UNIONs, wide select lists, long +//! literals, many statements, many-table joins); +//! * `closed_stdout`: a large result with standard output closed after a +//! few bytes (a broken pipe on every OS), then a `--write` statement that +//! must still run; +//! * `wide`: thousands of columns, a multi-megabyte field, a one-cell +//! `--write` that must change exactly one line. +//! +//! The oracle: the run ends within the time limit; the exit status is 0, 1 +//! or 2 (or a `.exit N` status); standard error never shows a panic, a +//! stack overflow or Rust's overflow text; a failed command line prints +//! exactly one line, `sqawk: ` (status 1), or clap's usage error +//! (status 2, `error: ...` and `Usage:`); a successful one prints nothing on +//! standard error; a piped REPL prints no banner or prompt, and each of its +//! error lines starts with `sqawk: `; no file appears in the working +//! directory, HOME or XDG_STATE_HOME (all fresh temp dirs), no temp file is +//! left next to the data, read-only files never change, and without +//! `--write` (or `.write` / `.save`) no input file changes. + +use std::collections::BTreeSet; +use std::ffi::OsString; +use std::fs; +use std::path::{Path, PathBuf}; +use std::time::Duration; + +use anyhow::{Context, Result}; + +use crate::diff::tables; +use crate::gen; +use crate::rng::Rng; +use crate::run::{crash_text, execute, Finished, Job, StdoutMode}; + +pub struct CliOptions { + pub iterations: usize, + pub only: Option, + pub timeout: Duration, +} + +/// csvfuzz.py's malformed inputs. +const MALFORMED: &[(&str, &[u8])] = &[ + ("empty.csv", b""), + ("newline_only.csv", b"\n"), + ("header_only.csv", b"a,b,c\n"), + ("header_only_nonl.csv", b"a,b,c"), + ("ragged_short.csv", b"a,b,c\n1,2\n3,4,5\n"), + ("ragged_long.csv", b"a,b\n1,2,3,4\n5,6\n"), + ("unclosed_quote.csv", b"a,b\n1,\"abc\n2,3\n"), + ("quote_mid.csv", b"a,b\n1,ab\"c\n2,\"x\"y\n"), + ("nul_bytes.csv", b"a,b\n1,x\x00y\n2,\x00\n"), + ("invalid_utf8.csv", b"a,b\n1,\xff\xfe\n2,caf\xe9\n"), + ("bom.csv", b"\xef\xbb\xbfid,name\n1,x\n"), + ("crlf_mixed.csv", b"a,b\r\n1,x\r\n2,y\n3,z\r\n"), + ("cr_only.csv", b"a,b\r1,x\r2,y\r"), + ("dup_header.csv", b"a,a,b\n1,2,3\n"), + ("empty_header.csv", b",b,\n1,2,3\n"), + ("space_header.csv", b"first name, age ,\"we,ird\"\n x , 5 ,q\n"), + ("blank_lines.csv", b"a,b\n1,2\n\n3,4\n\n"), + ("trailing_comma.csv", b"a,b,\n1,2,\n"), + ("embedded_nl.csv", b"a,b\n1,\"line1\nline2\"\n2,\"x\"\"y\"\n"), + ("only_quotes.csv", b"\"\"\n\"\"\n"), + ("numeric_header.csv", b"1,2,3\n4,5,6\n"), + ("comment_hash.csv", b"a,b\n#c,d\n1,2\n"), + ( + "huge_int.csv", + b"a\n99999999999999999999\n-9223372036854775809\n9223372036854775807\n1e999\nNaN\ninf\n-0\n0.1000\n", + ), + ("tabs.tsv", b"a\tb\n1\t\"q\tq\"\n# comment\n2\tx\n"), + ("noheader.txt", b"1 2 3\n4 5 6\n"), + ("case_dup.csv", b"id,ID\n1,2\n"), + ("quote_header.csv", b"\"a\"\"b\",c\n1,2\n"), + ("lone_cr_field.csv", b"a,b\n1,\"x\ry\"\n"), +]; + +/// fuzz.py's garbage-SQL tokens. +const TOKENS: &[&str] = &[ + "SELECT", + "FROM", + "WHERE", + "(", + ")", + ",", + "a", + "b", + "d", + "e", + "*", + "+", + "-", + "/", + "%", + "||", + "=", + "<", + ">", + "AND", + "OR", + "NOT", + "NULL", + "1", + "0", + "-9223372036854775808", + "9223372036854775807", + "'x'", + "''", + "1e999", + "GROUP BY", + "ORDER BY", + "HAVING", + "LIMIT", + "OFFSET", + "UNION", + "JOIN", + "ON", + "LEFT", + "CASE", + "WHEN", + "THEN", + "ELSE", + "END", + "IN", + "EXISTS", + "OVER", + "PARTITION BY", + "ROWS", + "count(*)", + "sum(", + "substr(", + "printf(", + "'%999999999d'", + "'%.999999999f'", + "INSERT INTO d VALUES", + "UPDATE d SET k =", + "DELETE FROM d", + ";", + "k", + "x", + "aid", + "DISTINCT", + "AS", + "CAST(", + "AS INTEGER)", + "lag(", + "ntile(", + "row_number()", + "date(", + "'+1e18 days'", + "zeroblob(", + "char(", + "1114112", + "-1", + "IS", + "BETWEEN", + "LIKE", + "ESCAPE", + "'%'", + "WITH", + "VALUES", + "(SELECT", + "RECURSIVE", + "BEGIN", + "COMMIT", + "ROLLBACK", + "TRUNCATE TABLE d", + "ALTER TABLE d ADD COLUMN z TEXT", + "CREATE TABLE", + "DROP TABLE d", + "FILTER", + "WINDOW", +]; + +/// fuzz.py's scalar calls at the limits. `{N}` is re-drawn from EXTREMES. +const LIMITS: &[&str] = &[ + "SELECT printf('%{N}d', 1)", + "SELECT printf('%.{N}f', 1)", + "SELECT printf('%{N}s', '')", + "SELECT substr('abc', {N}, {N})", + "SELECT char({N})", + "SELECT lpad('a', {N})", + "SELECT rpad('a', {N}, 'xy')", + "SELECT date('2024-01-01', '+{N} days')", + "SELECT date('2024-01-01', '+{N} seconds')", + "SELECT datetime({N}, 'unixepoch')", + "SELECT julianday({N})", + "SELECT strftime('%J', {N})", + "SELECT ntile({N}) OVER () FROM d", + "SELECT lag(k, {N}) OVER () FROM d", + "SELECT nth_value(k, {N}) OVER () FROM d", + "SELECT sum(k) OVER (ROWS BETWEEN {N} PRECEDING AND {N} FOLLOWING) FROM d", + "SELECT k FROM d LIMIT {N} OFFSET {N}", + "SELECT k FROM d LIMIT {N}", + "SELECT k FROM d ORDER BY {N}", + "SELECT k FROM d GROUP BY {N}", + "SELECT round(1.5, {N})", + "SELECT {N} % {N}, {N} / {N}, {N} * {N}, {N} + {N}, {N} - {N}", + "SELECT abs({N})", + "SELECT -({N})", + "SELECT CAST({N} AS INTEGER), CAST({N} AS REAL), CAST({N} AS TEXT)", + "SELECT hex(printf('%{N}d', 1))", + "SELECT length(printf('%{N}s', ''))", + "SELECT 'a' LIKE printf('%{N}s', '') || 'b'", + "SELECT instr(printf('%{N}s', 'a'), 'a')", + "SELECT group_concat(k, printf('%{N}s', '')) FROM d", + "SELECT unixepoch('9999-12-31 23:59:59', '+{N} second')", + "SELECT strftime('%s', '0000-01-01', '-{N} day')", + "SELECT date({N})", + "SELECT time({N})", + "SELECT zeroblob({N})", + "SELECT sum({N}) FROM d", + "SELECT avg({N}) FROM d", + "SELECT total({N}) FROM d", + "SELECT max({N}, {N}), min({N}, {N})", + "SELECT printf('%.*f', {N}, 1.5)", + "SELECT printf('%*d', {N}, 1)", + "SELECT substr(printf('%5000s', ''), {N})", + "WITH RECURSIVE c(n) AS (SELECT 1 UNION ALL SELECT n + 1 FROM c WHERE n < 1000) SELECT count(*) FROM c LIMIT {N}", + "SELECT * FROM d LIMIT 1 OFFSET {N}", + "SELECT round({N}, {N})", + "SELECT {N} BETWEEN {N} AND {N}", + "SELECT 1 FROM d a1, d a2, d a3, d a4, d a5, d a6 LIMIT {N}", +]; + +const EXTREMES: &[&str] = &[ + "9223372036854775807", + "-9223372036854775808", + "9223372036854775808", + "2147483648", + "2147483647", + "-2147483649", + "999999999", + "1000000001", + "1114112", + "55296", + "-1", + "0", + "1", + "1e308", + "-1e308", + "1e18", + "1e300", + "0.5", + "-0.0", + "NULL", + "'x'", + "'9223372036854775807'", +]; + +/// Valid statements over the data set's tables. +const GOOD_SQL: &[&str] = &[ + "SELECT * FROM a", + "SELECT COUNT(*) FROM b", + "SELECT 1", + "SELECT k, COUNT(*) FROM a GROUP BY k ORDER BY k", + "SELECT * FROM d ORDER BY k", + "SELECT a.aid, b.bid FROM a JOIN b ON a.aid = b.aid", + "UPDATE d SET wt = 99 WHERE k = 1", + "INSERT INTO d VALUES (77, 'new', 1)", + "DELETE FROM e WHERE eid = 1", + "BEGIN", + "COMMIT", + "ROLLBACK", + "SELECT * FROM people", + "SELECT name FROM people WHERE id = 2", + "ALTER TABLE dup ADD COLUMN z TEXT", + "TRUNCATE TABLE dup", + "SELECT * FROM nosuch", + "SELECT nosuchcol FROM a", + "CREATE TABLE tmp AS SELECT * FROM a", + "DROP TABLE dup", + "SELECT * FROM stdin", + "SELECT * FROM t", + "SELECT * FROM x", + "SELECT * FROM ro", + "UPDATE ro SET v = 'changed'", + "SELECT * FROM cmt", + "SELECT * FROM nohdr", +]; + +/// Separators: valid ones and the invalid ones clap must reject. +const SEPS: &[&str] = &[ + ",", "\t", "\\t", ";", "|", ":", " ", "x", "", "ab", "\"", "\n", "\r", "é", "\\n", "\\", +]; + +/// What one run is allowed to do and what it must look like. +struct Expect { + repl: bool, + verbose: bool, + /// Exit statuses beyond 0, 1 and 2 (`.exit N`). + extra_codes: BTreeSet, + /// Exactly this status. + code: Option, + /// Input files may change (`-w`, `.write`, `.save`). + writes: bool, + /// New entries allowed in the data directory. + new_files: BTreeSet, + /// A help / version run: stdout must not be empty. + info: bool, + /// Extra check after the run. + post: Option, +} + +enum Post { + /// The file must differ from these bytes (a write that had to happen). + Changed(PathBuf, Vec), + /// The file must differ from these bytes in exactly one line. + OneLine(PathBuf, Vec), +} + +impl Expect { + fn new() -> Self { + Expect { + repl: false, + verbose: false, + extra_codes: BTreeSet::new(), + code: None, + writes: false, + new_files: BTreeSet::new(), + info: false, + post: None, + } + } +} + +/// One iteration's run. +struct Case { + scenario: &'static str, + args: Vec, + stdin: Option>, + stdout: StdoutMode, + expect: Expect, +} + +/// The fixed data set, copied fresh into each iteration's directory. +fn write_data(dir: &Path) -> Result<()> { + fs::create_dir_all(dir)?; + gen::write_csv_tables(dir, &tables::edge())?; + fs::write(dir.join("people.tsv"), "id\tname\n1\tann\n2\tbob\n")?; + fs::write(dir.join("nohdr.txt"), "1,x\n2,y\n")?; + fs::write(dir.join("cmt.csv"), "#c\nid,v\n1,a\n#x\n2,b\n")?; + fs::write(dir.join("ro.csv"), "id,v\n1,a\n")?; + fs::create_dir_all(dir.join("k=v"))?; + fs::write(dir.join("k=v").join("t.csv"), "id,v\n1,a\n")?; + fs::create_dir_all(dir.join("adir"))?; + fs::create_dir_all(dir.join("bad"))?; + for (name, bytes) in MALFORMED { + fs::write(dir.join("bad").join(name), bytes)?; + } + Ok(()) +} + +fn copy_tree(from: &Path, to: &Path) -> Result<()> { + fs::create_dir_all(to)?; + for e in fs::read_dir(from)? { + let e = e?; + let p = e.path(); + let q = to.join(e.file_name()); + if p.is_dir() { + copy_tree(&p, &q)?; + } else { + fs::copy(&p, &q)?; + } + } + Ok(()) +} + +/// Every file under `dir`: relative path -> bytes. +fn snapshot(dir: &Path) -> Vec<(String, Option>)> { + let mut out = Vec::new(); + let mut stack = vec![dir.to_path_buf()]; + while let Some(d) = stack.pop() { + let Ok(rd) = fs::read_dir(&d) else { continue }; + for e in rd.flatten() { + let p = e.path(); + let rel = p + .strip_prefix(dir) + .map(|r| r.to_string_lossy().replace('\\', "/")) + .unwrap_or_default(); + if p.is_dir() { + out.push((rel + "/", None)); + stack.push(p); + } else { + out.push((rel, fs::read(&p).ok())); + } + } + } + out.sort(); + out +} + +fn set_readonly(path: &Path, ro: bool) { + if let Ok(meta) = fs::metadata(path) { + let mut p = meta.permissions(); + #[allow(clippy::permissions_set_readonly_false)] + p.set_readonly(ro); + let _ = fs::set_permissions(path, p); + } +} + +fn token_soup(r: &mut Rng, quotes: bool) -> String { + let n = r.range(2, 30) as usize; + let mut v = Vec::new(); + for _ in 0..n { + let t = *r.pick(TOKENS); + if !quotes && t.contains('\'') { + continue; + } + v.push(t); + } + v.join(" ") +} + +fn random_bytes(r: &mut Rng, max: usize) -> Vec { + let n = r.range(1, max as i64) as usize; + (0..n).map(|_| r.below(256) as u8).collect() +} + +fn limit_sql(r: &mut Rng) -> String { + let t = *r.pick(LIMITS); + let mut out = String::new(); + let mut rest = t; + while let Some(i) = rest.find("{N}") { + out.push_str(&rest[..i]); + out.push_str(r.pick_copy(EXTREMES)); + rest = &rest[i + 3..]; + } + out.push_str(rest); + out +} + +/// A very large statement (fuzz.py's nesting and size cases, sizes drawn). +fn huge_sql(r: &mut Rng) -> String { + let pick = |r: &mut Rng, xs: &[usize]| *r.pick(xs); + match r.below(20) { + 0 => { + let n = pick(r, &[40, 47, 48, 100, 1000, 20000]); + format!("SELECT {}1{}", "(".repeat(n), ")".repeat(n)) + } + 1 => format!( + "SELECT {}", + vec!["1"; pick(r, &[1000, 5000, 20000])].join("+") + ), + 2 => format!( + "SELECT count(*) FROM d WHERE {}", + vec!["k IS NOT NULL OR 1"; pick(r, &[100, 1000, 5000])].join(" AND ") + ), + 3 => format!("SELECT {}", vec!["'a'"; pick(r, &[1000, 10000])].join("||")), + 4 => format!("SELECT {}1", "NOT ".repeat(pick(r, &[100, 1000, 10000]))), + 5 => format!("SELECT {}1", "- ".repeat(pick(r, &[100, 1000, 10000]))), + 6 => { + let n = pick(r, &[100, 1000, 3000]); + format!("SELECT {}1{}", "coalesce(".repeat(n), ", 2)".repeat(n)) + } + 7 => { + let n = pick(r, &[100, 2000]); + format!( + "SELECT {}1{}", + "CASE WHEN 1 THEN ".repeat(n), + " END".repeat(n) + ) + } + 8 => { + let n = pick(r, &[30, 300, 2000]); + format!("SELECT {}1{}", "(SELECT ".repeat(n), ")".repeat(n)) + } + 9 => { + let n = pick(r, &[20, 100, 500]); + format!( + "SELECT * FROM {}d{}", + "(SELECT * FROM ".repeat(n), + ")".repeat(n) + ) + } + 10 => { + let n = pick(r, &[1000, 20000, 100000]); + let list: Vec = (0..n).map(|i| i.to_string()).collect(); + format!("SELECT count(*) FROM d WHERE k IN ({})", list.join(",")) + } + 11 => { + let n = pick(r, &[1000, 20000]); + let list: Vec = (0..n).map(|i| format!("'s{i}'")).collect(); + format!("SELECT count(*) FROM d WHERE label IN ({})", list.join(",")) + } + 12 => { + let n = pick(r, &[100, 1000, 3000]); + let kw = if r.chance(0.5) { + " UNION ALL " + } else { + " UNION " + }; + (0..n) + .map(|i| format!("SELECT {i}")) + .collect::>() + .join(kw) + } + 13 => { + let n = pick(r, &[1000, 5000, 20000]); + let cols: Vec = (0..n).map(|i| format!("k+{i}")).collect(); + format!("SELECT {} FROM d", cols.join(",")) + } + 14 => format!( + "SELECT length('{}')", + "x".repeat(pick(r, &[100_000, 1_000_000, 5_000_000])) + ), + 15 => vec!["SELECT 1"; pick(r, &[100, 1000, 5000])].join(";"), + 16 => { + let n = pick(r, &[100, 1000, 5000]); + let terms: Vec = (0..n).map(|i| format!("k = {i}")).collect(); + format!("SELECT count(*) FROM d WHERE {}", terms.join(" OR ")) + } + 17 => { + let n = pick(r, &[50, 200]); + format!("SELECT {}1{}", "abs(".repeat(n), ")".repeat(n)) + } + 18 => { + let n = pick(r, &[100, 2000]); + let ins: Vec = (0..n) + .map(|i| format!("INSERT INTO d VALUES ({i},'x',1)")) + .collect(); + format!("{}; SELECT count(*) FROM d", ins.join(";")) + } + _ => { + let n = pick(r, &[8, 16, 64]); + let from: Vec = (0..n).map(|i| format!("d t{i}")).collect(); + let wh: Vec = (0..n).map(|i| format!("t{i}.k = 1")).collect(); + format!( + "SELECT count(*) FROM {} WHERE {}", + from.join(", "), + wh.join(" AND ") + ) + } + } +} + +/// A random byte-level mutation of a valid CSV file. +fn mutate(r: &mut Rng, base: &[u8]) -> Vec { + let mut v = base.to_vec(); + let edits = r.range(1, 6); + for _ in 0..edits { + let at = if v.is_empty() { + 0 + } else { + r.below(v.len() + 1) + }; + match r.below(6) { + 0 => { + let b = *r.pick(&[b'"', b',', b'\r', b'\n', 0u8, 0xff, b'#', b' ', b'\t', 0xc3]); + v.insert(at, b); + } + 1 if !v.is_empty() => { + v.remove(at.min(v.len() - 1)); + } + 2 => v.truncate(at), + 3 => { + let line = b"1,2,3,4,5,6,7,8,9\n"; + for (i, b) in line.iter().enumerate() { + v.insert(at + i, *b); + } + } + 4 => { + let dup = v.clone(); + v.extend_from_slice(&dup); + } + _ => { + v.splice(at..at, b"\"\"\"".iter().copied()); + } + } + } + v +} + +fn os(s: impl Into) -> OsString { + s.into() +} + +fn build(r: &mut Rng, data: &Path, scratch: &Path) -> Result { + let roll = r.below(100); + let d = |name: &str| data.join(name).into_os_string(); + let mut expect = Expect::new(); + let mut args: Vec = Vec::new(); + let mut stdin = None; + let mut stdout = StdoutMode::Capture; + let edge_files = ["a.csv", "b.csv", "d.csv", "e.csv", "dup.csv"]; + let scenario: &'static str; + if roll < 30 { + scenario = "flags"; + let mut pieces: Vec> = Vec::new(); + let mut sql_from_stdin = false; + let mut table_from_stdin = false; + // No SQL or no operand is a usage error: keep those rare. + let nsql = if r.chance(0.08) { 0 } else { r.range(1, 3) }; + for _ in 0..nsql { + pieces.push(match r.below(10) { + 0..=4 => vec![os("-s"), os(*r.pick(GOOD_SQL))], + 5 => vec![os("-s"), os(token_soup(r, true))], + 6 => vec![os("-s"), os("")], + 7 => { + let f = scratch.join(format!("q{}.sql", r.below(1000))); + let mut text = String::new(); + for _ in 0..r.range(1, 4) { + text.push_str(r.pick_copy(GOOD_SQL)); + text.push_str(if r.chance(0.8) { ";\n" } else { "\n" }); + } + fs::write(&f, text)?; + vec![os("-f"), f.into_os_string()] + } + 8 => { + sql_from_stdin = true; + vec![os("-f"), os("-")] + } + _ => vec![ + os("-f"), + if r.chance(0.5) { + d("missing.sql") + } else { + d("adir") + }, + ], + }); + } + let nfiles = if r.chance(0.08) { 0 } else { r.range(1, 4) }; + for _ in 0..nfiles { + pieces.push(match r.below(24) { + 0..=3 | 16..=19 => vec![d(r.pick_copy(&edge_files))], + 4 | 20 => vec![d("people.tsv")], + 5 => vec![os(format!("x={}", data.join("a.csv").display()))], + 6 => vec![d("k=v/t.csv")], + 7 => vec![os("=x")], + 8 => { + table_from_stdin = true; + vec![os(if r.chance(0.5) { "-" } else { "t=-" })] + } + 9 => vec![d("missing.csv")], + 10 => vec![d("adir")], + 11 => vec![d("ro.csv")], + 12 => { + let (name, _) = *r.pick(MALFORMED); + vec![data.join("bad").join(name).into_os_string()] + } + 13 | 21 => vec![d("cmt.csv")], + 14 | 22 => vec![d("nohdr.txt")], + _ => vec![os(format!("{}=", data.join("a.csv").display()))], + }); + } + for _ in 0..r.range(0, 4) { + // Usage errors (exit 2) end a run before anything else is + // exercised, so the pieces that cause them are rare. + pieces.push(match r.below(30) { + 0..=3 => vec![ + os(r.pick_copy(&["-F", "--field-separator"])), + os(*r.pick(&SEPS[..7])), + ], + 4 => vec![ + os(r.pick_copy(&["-F", "--field-separator"])), + os(*r.pick(SEPS)), + ], + 5 | 6 => vec![os("--output-delimiter"), os(*r.pick(&SEPS[..7]))], + 7 => vec![os("--output-delimiter"), os(*r.pick(SEPS))], + 8 | 9 => vec![os("--no-output-header")], + 10 | 11 => vec![os("--no-header")], + 12 | 13 => vec![os("--comments")], + 14..=16 => vec![os(format!( + "--tabledef={}", + r.pick(&[ + "nohdr:n,v", + "nohdr:a,b,c", + "nosuch:x", + "bad", + ":x", + "nohdr:", + "a:p,q", + "nohdr:x,x" + ]) + ))], + 17 | 18 => { + expect.verbose = true; + vec![os("-v")] + } + 19..=22 => { + expect.writes = true; + vec![os(if r.chance(0.5) { "-w" } else { "--write" })] + } + 23 => vec![os("-i")], + 24 => vec![os("--bogus")], + 25 => vec![os("-s")], + 26 => vec![os("-F")], + 27 => { + expect.info = true; + vec![os(*r.pick(&["--help", "-h", "-V", "--version"]))] + } + _ => vec![os("--comments")], + }); + // A flag given twice is a usage error; once is enough. + let last = pieces.len() - 1; + let key = |p: &[OsString]| match p[0].to_str() { + Some("--write") => "-w".to_string(), + Some("--field-separator") => "-F".to_string(), + Some(s) => s.to_string(), + None => String::new(), + }; + let k = key(&pieces[last]); + if pieces[..last].iter().any(|p| key(p) == k) && !r.chance(0.05) { + pieces.pop(); + } + } + // Shuffle the pieces (Fisher-Yates), keeping each flag's value + // with it. + for i in (1..pieces.len()).rev() { + let j = r.below(i + 1); + pieces.swap(i, j); + } + for p in pieces { + args.extend(p); + } + if sql_from_stdin || table_from_stdin || r.chance(0.2) { + stdin = Some(if sql_from_stdin && (!table_from_stdin || r.chance(0.5)) { + format!("{};\n", r.pick(GOOD_SQL)).into_bytes() + } else { + b"a,b\n1,2\n3,4\n".to_vec() + }); + } + // `-i` reads its commands from stdin. + if args.iter().any(|a| a == "-i") { + expect.repl = true; + if stdin.is_none() { + stdin = Some(b"SELECT 1;\n".to_vec()); + } + } + } else if roll < 50 { + scenario = "repl"; + expect.repl = true; + args.push(os("-i")); + for f in edge_files.iter().take(r.range(1, 5) as usize) { + args.push(d(f)); + } + if r.chance(0.3) { + args.push(d("people.tsv")); + } + if r.chance(0.15) { + expect.verbose = true; + args.push(os("-v")); + } + if r.chance(0.15) { + args.push(os("-w")); + expect.writes = true; + } + if r.chance(0.1) { + args.push(os("--no-output-header")); + } + let mut script = String::new(); + let n = r.range(1, 15); + for _ in 0..n { + match r.below(20) { + 0..=6 => { + let sql = *r.pick(GOOD_SQL); + // Split the statement over lines at random spaces. + let mut line = String::new(); + for w in sql.split(' ') { + line.push_str(w); + line.push(if r.chance(0.2) { '\n' } else { ' ' }); + } + script.push_str(line.trim_end()); + script.push_str(";\n"); + } + 7 => { + script.push_str(&token_soup(r, false)); + script.push_str(";\n"); + } + 8 => { + let cmd = match r.below(6) { + 0 => format!(".load {}", data.join("people.tsv").display()), + 1 => format!(".load p2={}", data.join("cmt.csv").display()), + 2 => ".load".to_string(), + 3 => ".load /nonexistent/x.csv".to_string(), + 4 => format!(".load {}", data.join("adir").display()), + _ => format!( + ".load {}", + data.join("bad").join(r.pick(MALFORMED).0).display() + ), + }; + script.push_str(&cmd); + script.push('\n'); + } + 9 => { + let cmd = match r.below(4) { + 0 => format!(".cd {}", data.display()), + 1 => ".cd".to_string(), + 2 => ".cd /nonexistent/dir".to_string(), + _ => format!(".cd {}", data.join("a.csv").display()), + }; + script.push_str(&cmd); + script.push('\n'); + } + 10 => { + expect.writes = true; + script.push_str(r.pick_copy(&[ + ".save", + ".save d", + ".save nosuch", + ".write on", + ".write off", + ".write", + ])); + script.push('\n'); + } + 11 => { + script.push_str(r.pick_copy(&[ + ".tables", + ".tables a%", + ".tables zz", + ".schema", + ".schema a", + ".schema nosuch", + ".help", + ".version", + ".show", + ".show tables", + ".stats on", + ".stats off", + ".stats bogus", + ".changes on", + ".changes off", + ".changes x", + ".print hello world", + ".bogus", + ".", + ".TABLES", + ])); + script.push('\n'); + } + 12 => { + script.push_str(r.pick_copy(&[ + "BEGIN;\n", + "COMMIT;\n", + "ROLLBACK;\n", + "END;\n", + ])); + } + 13 => { + script.push_str(&format!( + "CREATE TABLE nt (id INT, v TEXT) LOCATION '{}';\nINSERT INTO nt VALUES (1, 'x');\n", + data.join("newloc.csv").display() + )); + expect.new_files.insert("newloc.csv".to_string()); + } + 14 => script.push_str("\n \n"), + 15 => script.push_str("SELECT 1; SELECT 2;\n"), + 16 => script.push_str("SELECT 'a;b', \"k\" FROM d WHERE k = 1;\n"), + _ => script.push_str(&format!("{};\n", limit_sql(r))), + } + } + match r.below(6) { + 0 => { + let code = *r.pick(&[ + "0", + "1", + "3", + "-1", + "255", + "256", + "300", + "abc", + "99999999999999999999", + "", + ]); + if let Ok(c) = code.parse::() { + expect.extra_codes.insert(c.rem_euclid(256) as i32); + } + script.push_str(&format!(".exit {code}\n")); + // Lines after .exit never run. + script.push_str("SELECT 'after exit';\n"); + } + 1 => script.push_str(".quit\n"), + 2 => script.push_str("SELECT * FROM d WHERE k = 1"), + 3 => script.push_str("SELECT 'unterminated"), + _ => {} + } + if r.chance(0.1) { + script = script.replace('\n', "\r\n"); + } + stdin = Some(script.into_bytes()); + } else if roll < 63 { + scenario = "malformed"; + let (name, bytes): (String, Vec) = if r.chance(0.5) { + let (n, b) = *r.pick(MALFORMED); + (n.to_string(), b.to_vec()) + } else { + let base = fs::read(data.join("a.csv"))?; + let ext = *r.pick(&["csv", "tsv", "txt"]); + (format!("m.{ext}"), mutate(r, &base)) + }; + let path = scratch.join(&name); + fs::write(&path, &bytes)?; + let table = name.split('.').next().unwrap_or("m").to_string(); + if r.chance(0.3) { + args.push(os("--comments")); + } + if r.chance(0.2) { + args.push(os("--no-header")); + } + if r.chance(0.15) { + args.push(os("-F")); + args.push(os(*r.pick(&[",", "\\t", ";", " "]))); + } + let sql = match r.below(4) { + 0 => format!("SELECT * FROM \"{table}\""), + 1 => format!("SELECT count(*) FROM \"{table}\""), + 2 => { + expect.writes = true; + args.push(os("-w")); + format!("DELETE FROM \"{table}\" WHERE rowid = 1; INSERT INTO \"{table}\" SELECT * FROM \"{table}\" LIMIT 1") + } + _ => { + expect.writes = true; + args.push(os("-w")); + format!("UPDATE \"{table}\" SET \"{}\" = 'X'", 'a') + } + }; + args.push(os("-s")); + args.push(os(sql)); + args.push(path.into_os_string()); + } else if roll < 75 { + scenario = "garbage"; + let sql: Vec = if r.chance(0.33) { + random_bytes(r, 80) + } else { + token_soup(r, true).into_bytes() + }; + let text_ok = std::str::from_utf8(&sql).is_ok() && !sql.contains(&0); + if text_ok && r.chance(0.5) { + args.push(os("-s")); + args.push(os(String::from_utf8(sql).expect("checked"))); + } else if r.chance(0.5) { + let f = scratch.join("garbage.sql"); + fs::write(&f, &sql)?; + args.push(os("-f")); + args.push(f.into_os_string()); + } else { + args.push(os("-f")); + args.push(os("-")); + stdin = Some(sql); + } + for f in edge_files { + args.push(d(f)); + } + } else if roll < 83 { + scenario = "limits"; + args.push(os("-s")); + args.push(os(limit_sql(r))); + args.push(d("d.csv")); + } else if roll < 89 { + scenario = "huge"; + let sql = huge_sql(r); + if r.chance(0.7) { + let f = scratch.join("huge.sql"); + fs::write(&f, &sql)?; + args.push(os("-f")); + args.push(f.into_os_string()); + args.push(d("d.csv")); + } else { + expect.repl = true; + args.push(os("-i")); + args.push(d("d.csv")); + stdin = Some(format!("{sql};\n.exit\n").into_bytes()); + } + } else if roll < 95 { + scenario = "closed_stdout"; + let n = *r.pick(&[1000, 100_000, 300_000]); + let close_after = *r.pick(&[0usize, 1, 100, 10_000]); + stdout = StdoutMode::CloseAfter(close_after); + let big = format!( + "WITH RECURSIVE c(n) AS (SELECT 1 UNION ALL SELECT n + 1 FROM c WHERE n < {n}) SELECT n, 'xxxxxxxxxxxxxxxxxxxx' AS pad FROM c" + ); + if r.chance(0.7) { + args.push(os("-s")); + args.push(os(big)); + args.push(os("-s")); + args.push(os("UPDATE d SET wt = 4242 WHERE k = 1")); + args.push(os("-w")); + // -v traces every instruction: only on the small result. + if n == 1000 && r.chance(0.5) { + expect.verbose = true; + args.push(os("-v")); + } + args.push(d("d.csv")); + expect.writes = true; + expect.code = Some(0); + let before = fs::read(data.join("d.csv"))?; + expect.post = Some(Post::Changed(data.join("d.csv"), before)); + } else { + expect.repl = true; + args.push(os("-i")); + args.push(d("d.csv")); + stdin = Some(format!("{big};\n.tables\n.help\nSELECT 1;\n.exit 3\n").into_bytes()); + expect.code = Some(3); + expect.extra_codes.insert(3); + } + } else { + scenario = "wide"; + let roll = r.below(3); + if roll == 0 { + let len = *r.pick(&[1_000_000usize, 4_000_000]); + let mut b = b"a,b\n1,".to_vec(); + b.extend(std::iter::repeat_n(b'x', len)); + b.push(b'\n'); + let p = scratch.join("hugefield.csv"); + fs::write(&p, b)?; + args.push(os("-s")); + args.push(os("SELECT length(b) FROM hugefield")); + args.push(p.into_os_string()); + } else { + let ncol = *r.pick(&[500usize, 1000, 2000, 10000]); + let hdr: Vec = (0..ncol).map(|i| format!("c{i}")).collect(); + let mut text = hdr.join(","); + text.push('\n'); + for row in 0..20usize { + let cells: Vec = (0..ncol).map(|i| (row * i).to_string()).collect(); + text.push_str(&cells.join(",")); + text.push('\n'); + } + let p = scratch.join("wide.csv"); + fs::write(&p, &text)?; + if roll == 1 { + let sql = *r.pick(&[ + "SELECT count(*), sum(c999) FROM wide", + "SELECT * FROM wide", + "SELECT c1 % 3, count(*) FROM wide GROUP BY 1", + "SELECT count(*) FROM wide x JOIN wide y ON x.c5 = y.c5", + ]); + args.push(os("-s")); + args.push(os(sql)); + } else { + expect.writes = true; + args.push(os("-w")); + args.push(os("-s")); + args.push(os(format!( + "UPDATE wide SET c{} = 0 WHERE c1 = 3", + ncol / 2 + ))); + expect.code = Some(0); + expect.post = Some(Post::OneLine(p.clone(), text.into_bytes())); + } + args.push(p.into_os_string()); + } + } + Ok(Case { + scenario, + args, + stdin, + stdout, + expect, + }) +} + +fn show_args(args: &[OsString]) -> String { + args.iter() + .map(|a| { + let s = a.to_string_lossy(); + let s: String = if s.chars().count() > 120 { + format!( + "{}...({} chars)", + s.chars().take(100).collect::(), + s.chars().count() + ) + } else { + s.into_owned() + }; + if s.is_empty() + || s.contains([ + ' ', '\t', '\n', '\'', '"', ';', '*', '(', ')', '|', '<', '>', + ]) + { + format!("{s:?}") + } else { + s + } + }) + .collect::>() + .join(" ") +} + +/// Problems with a finished run; the first element says crash or bad. +fn judge(f: &Finished, e: &Expect, timeout: Duration) -> (bool, Vec) { + let mut crash = false; + let mut bad = Vec::new(); + let stderr = f.stderr_text(); + let stdout = String::from_utf8_lossy(&f.stdout); + if f.timed_out { + return ( + true, + vec![format!("hang: no exit within {}s", timeout.as_secs())], + ); + } + let Some(code) = f.code else { + return ( + true, + vec![format!("killed by a signal; stderr {:?}", stderr.trim())], + ); + }; + if let Some(t) = crash_text(&stderr) { + crash = true; + bad.push(format!("stderr contains {t:?}: {}", tail(&stderr))); + } + if let Some(want) = e.code { + if code != want { + bad.push(format!( + "exit status {code}, want {want}; stderr {}", + tail(&stderr) + )); + } + } else if !(0..=2).contains(&code) && !e.extra_codes.contains(&code) { + bad.push(format!("exit status {code}; stderr {}", tail(&stderr))); + } + let lines: Vec<&str> = stderr.lines().collect(); + if e.repl && !e.verbose { + for l in &lines { + if code == 2 && (l.starts_with("error:") || !l.starts_with("sqawk: ")) { + // A usage error before the REPL starts is clap's. + continue; + } + if !l.starts_with("sqawk: ") { + bad.push(format!("REPL stderr line not `sqawk: `: {l:?}")); + break; + } + } + if stdout.contains("Welcome to Sqawk") || stdout.contains("sqawk> ") { + bad.push("piped REPL printed its banner or prompt".to_string()); + } + } else if !e.repl { + match code { + 0 => { + if !e.verbose && !stderr.is_empty() { + bad.push(format!("success with stderr output: {}", tail(&stderr))); + } + if e.info && f.stdout.is_empty() { + bad.push("help / version printed nothing".to_string()); + } + } + 1 => { + let last = lines.last().copied().unwrap_or(""); + if !last.starts_with("sqawk: ") { + bad.push(format!("error line not `sqawk: `: {}", tail(&stderr))); + } else if !e.verbose && lines.len() != 1 { + bad.push(format!("error is {} lines: {}", lines.len(), tail(&stderr))); + } + } + // clap: `error: ...`, then the usage or a pointer to --help. + 2 if !stderr.starts_with("error: ") + || !(stderr.contains("Usage:") || stderr.contains("try '--help'")) => + { + bad.push(format!( + "status 2 without clap's usage error: {}", + tail(&stderr) + )); + } + _ => {} + } + } + (crash, bad) +} + +fn tail(s: &str) -> String { + let t = s.trim(); + let n = t.chars().count(); + if n > 400 { + format!("{:?}", t.chars().skip(n - 400).collect::()) + } else { + format!("{t:?}") + } +} + +/// Lines of `a` and `b` that differ (and whether the line counts match). +fn line_diff(a: &[u8], b: &[u8]) -> (usize, bool) { + let la: Vec<&[u8]> = a.split(|&c| c == b'\n').collect(); + let lb: Vec<&[u8]> = b.split(|&c| c == b'\n').collect(); + let n = la.iter().zip(&lb).filter(|(x, y)| x != y).count(); + (n, la.len() == lb.len()) +} + +/// One iteration: (crashed, problems, description with run time). +fn one( + seed: u64, + it: usize, + sqawk: &Path, + base: &Path, + timeout: Duration, +) -> Result<(bool, Vec, String)> { + let mut r = Rng::stream(seed, it as u64 + 5000); + let root = base.join(format!("it{it}")); + let data = root.join("data"); + let scratch = root.join("scratch"); + let cwd = root.join("cwd"); + let home = root.join("home"); + let state = root.join("state"); + copy_tree(&base.join("data"), &data)?; + for d in [&scratch, &cwd, &home, &state] { + fs::create_dir_all(d)?; + } + let ro = data.join("ro.csv"); + set_readonly(&ro, true); + let ro_bytes = fs::read(&ro)?; + let case = build(&mut r, &data, &scratch)?; + let desc = format!( + "[{}] sqawk {}{}", + case.scenario, + show_args(&case.args), + case.stdin + .as_ref() + .map(|s| { + let t = String::from_utf8_lossy(s); + let t: String = t.chars().take(600).collect(); + format!("\n stdin: {t:?}") + }) + .unwrap_or_default() + ); + let before = snapshot(&data); + let mut job = Job::new(sqawk, case.args.clone()); + job.stdin = case.stdin.clone(); + job.cwd = Some(cwd.clone()); + job.limit = timeout; + job.stdout = case.stdout; + job.env = vec![ + ("HOME".to_string(), home.clone().into_os_string()), + ("USERPROFILE".to_string(), home.clone().into_os_string()), + ("XDG_STATE_HOME".to_string(), state.clone().into_os_string()), + ("RUST_BACKTRACE".to_string(), OsString::from("0")), + ]; + let f = execute(&job); + if let Some(e) = &f.spawn_error { + anyhow::bail!("{e}"); + } + let (crash, mut bad) = judge(&f, &case.expect, timeout); + // Files. + for (label, d) in [ + ("working directory", &cwd), + ("HOME", &home), + ("XDG_STATE_HOME", &state), + ] { + let left = snapshot(d); + if !left.is_empty() { + bad.push(format!( + "files left in {label}: {:?}", + left.iter().map(|x| &x.0).collect::>() + )); + } + } + if fs::read(&ro).ok().as_deref() != Some(&ro_bytes[..]) { + bad.push("read-only ro.csv changed".to_string()); + } + let after = snapshot(&data); + let before_names: BTreeSet<&String> = before.iter().map(|x| &x.0).collect(); + for (name, bytes) in &after { + if !before_names.contains(name) { + let base_name = name.rsplit('/').next().unwrap_or(name); + if !case.expect.new_files.contains(base_name) { + bad.push(format!("new file in the data directory: {name}")); + } + continue; + } + if !case.expect.writes { + let old = before + .iter() + .find(|x| &x.0 == name) + .and_then(|x| x.1.as_ref()); + if old != bytes.as_ref() { + bad.push(format!("{name} changed without --write")); + } + } + } + for (name, _) in &before { + if !after.iter().any(|x| &x.0 == name) && !case.expect.writes { + bad.push(format!("{name} disappeared")); + } + } + let scratch_left: Vec = snapshot(&scratch) + .into_iter() + .map(|x| x.0) + .filter(|n| n.rsplit('/').next().unwrap_or(n).starts_with(".tmp")) + .collect(); + if !scratch_left.is_empty() { + bad.push(format!("temp files left: {scratch_left:?}")); + } + match &case.expect.post { + Some(Post::Changed(p, old)) if f.code == Some(0) => { + if fs::read(p).ok().as_deref() == Some(&old[..]) { + bad.push(format!( + "{} not written after the reader closed stdout", + p.file_name().unwrap_or_default().to_string_lossy() + )); + } + } + Some(Post::OneLine(p, old)) if f.code == Some(0) => { + let now = fs::read(p).unwrap_or_default(); + let (n, same_count) = line_diff(old, &now); + if n != 1 || !same_count { + bad.push(format!( + "one-cell --write changed {n} lines (line count kept: {same_count})" + )); + } + } + _ => {} + } + set_readonly(&ro, false); + let _ = fs::remove_dir_all(&root); + let first_err: String = f + .stderr_text() + .lines() + .next() + .unwrap_or("") + .chars() + .take(160) + .collect(); + let desc = format!( + "{:.2}s exit {} {desc}{}", + f.elapsed.as_secs_f64(), + f.code.map_or("-".to_string(), |c| c.to_string()), + if first_err.is_empty() { + String::new() + } else { + format!("\n stderr: {first_err}") + } + ); + Ok((crash, bad, desc)) +} + +pub fn run(seed: u64, sqawk: &Path, opts: &CliOptions, verbose: bool) -> Result { + let base = tempfile::Builder::new().prefix("tsq-cli-").tempdir()?; + write_data(&base.path().join("data")).context("write the data set")?; + let (mut ok, mut bad, mut crash) = (0usize, 0usize, 0usize); + let range: Vec = match opts.only { + Some(n) => vec![n], + None => (0..opts.iterations).collect(), + }; + for it in range { + let (is_crash, problems, desc) = one(seed, it, sqawk, base.path(), opts.timeout) + .with_context(|| format!("iteration {it}"))?; + if problems.is_empty() { + ok += 1; + if verbose || opts.only.is_some() { + println!("ok it={it} {desc}"); + } + } else { + if is_crash { + crash += 1; + } else { + bad += 1; + } + println!( + "{} it={it} (tsq cli --seed {seed} --only {it})\n {desc}\n {}", + if is_crash { "CRASH" } else { "BAD" }, + problems.join("\n ") + ); + } + } + // A read-only file left behind would stop the temp dir's removal on + // Windows. + set_readonly(&base.path().join("data").join("ro.csv"), false); + println!("CLI SUMMARY {{ok: {ok}, bad: {bad}, crash: {crash}}}"); + Ok(if bad + crash > 0 { 1 } else { 0 }) +} diff --git a/tsq/src/corpus.rs b/tsq/src/corpus.rs new file mode 100644 index 0000000..af161ac --- /dev/null +++ b/tsq/src/corpus.rs @@ -0,0 +1,112 @@ +//! `tsq corpus`: hand-written test files, judged by the same oracle and +//! renderer as `tsq diff`. +//! +//! Format (from the retired Python harness, harness.py): one test per line; blank +//! lines and lines starting with `--` are skipped; a line prefixed `O:` is +//! compared in order, any other as a multiset of rows. A line may hold +//! several `;`-separated statements: both engines run all of them and every +//! result set is compared (header included). A comment line +//! `-- tsq-tables: NAME` selects the built-in table set the file runs +//! against (`harness` when absent; `--tables` overrides). + +use std::fs; +use std::path::{Path, PathBuf}; +use std::time::Duration; + +use anyhow::{bail, Context, Result}; + +use crate::diff::oracle::{split_statements, Oracle}; +use crate::diff::{describe, run_all, tables, Cat, Env, Script, Tally}; + +pub struct CorpusOptions { + pub files: Vec, + pub tables: Option, + pub jobs: usize, + pub timeout: Duration, + pub keep_going: bool, +} + +pub struct Test { + pub file: String, + pub line: usize, + pub ordered: bool, + pub sql: String, +} + +/// The tests in `text` and the table set it asks for. +pub fn parse(file: &str, text: &str) -> (Vec, Option) { + let mut tests = Vec::new(); + let mut set = None; + for (n, line) in text.lines().enumerate() { + let q = line.trim(); + if let Some(rest) = q.strip_prefix("-- tsq-tables:") { + set = Some(rest.trim().to_string()); + continue; + } + if q.is_empty() || q.starts_with("--") { + continue; + } + let (ordered, sql) = match q.strip_prefix("O:") { + Some(r) => (true, r.trim()), + None => (false, q), + }; + tests.push(Test { + file: file.to_string(), + line: n + 1, + ordered, + sql: sql.to_string(), + }); + } + (tests, set) +} + +pub fn run(sqawk: &Path, opts: &CorpusOptions, verbose: bool) -> Result { + let mut groups: Vec<(String, Vec)> = Vec::new(); + for f in &opts.files { + let text = fs::read_to_string(f).with_context(|| format!("read {}", f.display()))?; + let name = f + .file_name() + .map(|n| n.to_string_lossy().into_owned()) + .unwrap_or_default(); + let (tests, set) = parse(&name, &text); + let set = opts + .tables + .clone() + .or(set) + .unwrap_or_else(|| "harness".to_string()); + match groups.iter_mut().find(|(s, _)| *s == set) { + Some((_, v)) => v.extend(tests), + None => groups.push((set, tests)), + } + } + let mut tally = Tally::default(); + for (set, tests) in &groups { + let Some(tabs) = tables::builtin(set) else { + bail!("unknown table set {set:?} (harness, edge)"); + }; + let tmp = tempfile::Builder::new().prefix("tsq-corpus-").tempdir()?; + crate::gen::write_csv_tables(tmp.path(), &tabs)?; + let oracle = Oracle::new(&tabs)?; + let schema = tables::schema_of(&tabs, &oracle)?; + let env = Env::new(sqawk, tmp.path(), &tabs, &schema, opts.timeout); + let scripts: Vec