diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e6e0a93..ff8e983 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -46,20 +46,21 @@ jobs: components: clippy - uses: Swatinem/rust-cache@v2 # Without dev-dependencies, which cargo test and clippy --all-targets - # merge in: this is the feature set users actually build. + # merge in: this is the feature set users actually build. --workspace + # adds tsq/, the unpublished test-data generator crate. - name: Build - run: cargo build --locked - # On Windows too, not just Linux: src/tsq.rs and src/storage/mmap.rs have - # cfg(unix) branches whose non-unix side is otherwise never linted. macOS - # is unix like Linux, so it adds nothing. + run: cargo build --locked --workspace + # On Windows too, not just Linux: tsq/src/main.rs has cfg(unix) branches + # whose non-unix side is otherwise never linted. macOS is unix like + # Linux, so it adds nothing. - name: Clippy if: runner.os != 'macOS' - run: cargo clippy --locked --all-targets -- -D warnings + run: cargo clippy --locked --workspace --all-targets -- -D warnings # Run even when Clippy failed, so a lint (often a new one from a stable # toolchain release) does not hide the test results on every OS. - name: Test if: ${{ !cancelled() }} - run: cargo test --locked + run: cargo test --locked --workspace test-release: name: Test (release) @@ -72,7 +73,7 @@ jobs: - uses: dtolnay/rust-toolchain@stable - uses: Swatinem/rust-cache@v2 - name: Test - run: cargo test --locked --release + run: cargo test --locked --workspace --release fmt: name: Format @@ -107,7 +108,8 @@ jobs: with: toolchain: ${{ steps.msrv.outputs.version }} - uses: Swatinem/rust-cache@v2 - # Library and binaries only: rust-version is a promise to consumers, and - # dev-dependencies are not part of it. + # Binaries only: rust-version is a promise to people building sqawk, and + # dev-dependencies are not part of it. --workspace includes tsq/, which + # declares the same rust-version. - name: Check build at MSRV - run: cargo check --locked + run: cargo check --locked --workspace diff --git a/CLAUDE.md b/CLAUDE.md index 1053081..ab209cd 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -9,7 +9,7 @@ This file provides guidance to Claude Code (claude.ai/code) when working with co cargo build # Run all tests -cargo test +cargo test --workspace # sqawk + the tsq dev crate # Run a specific test module cargo test --test mod basic # Basic end-to-end tests @@ -28,6 +28,9 @@ cargo run -- -s "SELECT * FROM data" data.csv # Run in interactive REPL mode cargo run -- --interactive data.csv + +# Generate test data with tsq (unpublished workspace crate in tsq/) +cargo run -p tsq -- --seed 42 --rows 1000 --output-dir /tmp/sqawk-test ``` ## Architecture Overview @@ -55,13 +58,44 @@ SQL execution uses a bytecode VM inspired by SQLite's architecture: compiler (the analogue of SQLite's `sqlite3ExprCode`) used from the SELECT list, WHERE, HAVING, ORDER BY and SET clauses alike, plus `NameCtx`, which carries the FROM sources a name can resolve against +- `compiler_select.rs`: the SELECT driver -- `compile_query`, compound + operators (UNION/INTERSECT/EXCEPT via keyed ephemeral tables) and the single + output stage (ORDER BY with hidden sort keys, DISTINCT, LIMIT/OFFSET) every + SELECT core ends in - `compiler_aggregate.rs`: aggregates, GROUP BY, HAVING -- `compiler_join.rs`: joins +- `compiler_from.rs`: the FROM planner used by every SELECT -- tables, comma + lists and join chains become nested loop levels (ON per level, WHERE after + outer-join NULL-extension, hash join on equi-conditions); the projection / + aggregate stage is a body callback emitted at the innermost level; a + FROM-less SELECT is a plan with zero sources (one row) - `compiler_dml.rs`: INSERT / UPDATE / DELETE - `compiler_ddl.rs`: CREATE / DROP / ALTER / TRUNCATE -- `compiler_window.rs`: window functions +- `compiler_window.rs`: the window stage -- finished rows (every column, or + a grouped query's keys and aggregates) plus window keys/arguments go into + an ephemeral table; one `WindowCompute` per distinct PARTITION BY/ORDER BY + sorts it and fills each function's result column (`src/vm/window.rs` is + the run-time half); the output stage reads rows back with each window call + bound like an aggregate, so window calls work inside any expression and in + ORDER BY +- `compiler_subquery.rs`: scalar / EXISTS / IN subqueries, compiled by the + same query compiler as fragments of the enclosing program (`Once` when + uncorrelated; correlation found by `NameCtx` outward resolution) +- `compiler_check.rs`: rejects every parsed clause sqawk does not implement + (RETURNING, ON CONFLICT, UPDATE FROM, WITH, QUALIFY, ...) before + compilation; INSERT/UPDATE/DELETE/Query/Select are destructured + exhaustively so a new sqlparser field fails to compile until handled - `ast_compat.rs`: thin accessors normalizing sqlparser AST shapes - `engine.rs`: Executes bytecode instructions +- `arith.rs`: SQLite numeric semantics -- text-to-number coercion, i64 + arithmetic with overflow to real, CAST, truth values +- `func.rs`: the scalar function table (`FuncDef`: name, arity, body) run by + the `Function` opcode (SQLite's OP_Function); arity is checked at compile + time. Lazy functions (COALESCE, IFNULL, IIF, NULLIF) are compiled to jumps +- `datetime.rs`: DATE/TIME/DATETIME/STRFTIME/... (SQLite time strings and + modifiers, UTC) +- `like.rs`: the LIKE matcher (no regex; compiled pattern cached per + instruction) +- `agg.rs`: aggregate accumulators shared by AggStep/AggFinal and windows Each statement is compiled and executed separately, so a statement observes the effects of the ones before it. @@ -69,15 +103,22 @@ effects of the ones before it. ### File Handling - **`file_handler.rs`**: Manages loading files into database tables -- **`csv_handler.rs`**: Standard CSV file parsing (comma-separated) -- **`delim_handler.rs`**: Custom delimiter support via `-F` flag (TSV, colon-separated, etc.) +- **`csv_handler.rs`**: The single loader (`read_table`, built on the `csv` + crate) for every delimiter and for stdin, plus writeback. `.csv` files always + have a header; other files keep the header guess and `#`-comment skipping. + Fields load as Integer/Float only when the text round-trips exactly, so + untouched cells write back byte-for-byte ### SQL Features -Expression compilation, including all scalar and string functions, lives in -`src/vm/compiler.rs` behind `code_expr`. Aggregates are compiled in -`src/vm/compiler_aggregate.rs` and executed by the `AggStep`/`AggFinal` -opcodes; `src/aggregate.rs` retains only the function-name lookup. +Expression compilation lives in `src/vm/compiler.rs` behind `code_expr`; +scalar functions are looked up in `src/vm/func.rs` and called through the +`Function` opcode. Aggregates (including FILTER and GROUP_CONCAT's separator) +are compiled in `src/vm/compiler_aggregate.rs` and executed by the +`AggStep`/`AggFinal` opcodes over `src/vm/agg.rs` accumulators; +`src/aggregate.rs` retains only the function-name lookup and arity. Semantics +follow SQLite 3.45; deliberate differences are listed in +`doc/sql_reference.md` ("Differences from SQLite"). ### Safe Writeback Model @@ -87,6 +128,7 @@ By default, all modifications remain in memory only. The `--write` flag must be Tests are in `tests/` organized by functionality: - `basic/`, `comparison/`, `join/`, `join_on/`, `order_by/`, `update/` +- `unsupported/`: parsed-but-unimplemented syntax must error and modify nothing - `aggregate/`, `alias/`, `distinct/`, `group_by/`, `limit_offset/` - `delimiter/`, `csv_handler/`, `string_functions/`, `repl/` - `helpers/`: Test utilities @@ -103,4 +145,10 @@ Two suites carry the correctness contract: defects. Run with `cargo test --test mod defects -- --ignored`. Remove the `#[ignore]` when a defect is fixed; the list only shrinks. +`tests/sqlite_diff/harness.py` is a differential tester against SQLite (via +python3's `sqlite3` module), not part of `cargo test`. Run +`cargo build --release && python3 -I tests/sqlite_diff/harness.py +tests/sqlite_diff/queries.sql tests/sqlite_diff/queries2.sql`; the SUMMARY +line counts pass / DIFF (wrong answer) / sqawk_err. + Tests use `assert_cmd` for CLI testing and `tempfile` for temporary test data. diff --git a/Cargo.lock b/Cargo.lock index c408721..a542a44 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -334,15 +334,6 @@ version = "2.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98" -[[package]] -name = "memmap2" -version = "0.9.11" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d1219ed1b7f229ee7104d281dd01d6802fe28bb6e95d292942c4daacdeb798c0" -dependencies = [ - "libc", -] - [[package]] name = "nibble_vec" version = "0.1.0" @@ -604,8 +595,6 @@ dependencies = [ "assert_cmd", "clap", "csv", - "libc", - "memmap2", "predicates", "regex", "rustyline", @@ -683,6 +672,17 @@ version = "0.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8f50febec83f5ee1df3015341d8bd429f2d1cc62bcba7ea2076759d315084683" +[[package]] +name = "tsq" +version = "0.1.0" +dependencies = [ + "anyhow", + "assert_cmd", + "clap", + "libc", + "tempfile", +] + [[package]] name = "unicode-ident" version = "1.0.24" diff --git a/Cargo.toml b/Cargo.toml index 860c58e..1aae302 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -7,11 +7,7 @@ description = "An SQL-based command-line tool for processing delimiter-separated license = "MIT" repository = "https://github.com/jgarzik/sqawk" homepage = "https://github.com/jgarzik/sqawk" -documentation = "https://docs.rs/sqawk" readme = "README.md" -# `cargo run` and `cargo install --path .` would otherwise be ambiguous across -# the two [[bin]] targets below. -default-run = "sqawk" keywords = ["csv", "tsv", "sql", "delimited", "awk"] categories = ["command-line-utilities", "text-processing", "parser-implementations"] # Minimum supported Rust version. @@ -37,8 +33,6 @@ sqlparser = "0.62" csv = "1" anyhow = "1" regex = "1" -memmap2 = "0.9" -libc = "0.2" tempfile = "3" rustyline = "15" @@ -58,6 +52,11 @@ debug = true name = "sqawk" path = "src/main.rs" -[[bin]] -name = "tsq" -path = "src/tsq.rs" +# `tsq`, the test-data generator, is a separate workspace member with +# `publish = false`: it builds and runs locally (`cargo run -p tsq`) but is +# not part of the published sqawk crate. `cargo package` skips nested crates. +# Only sqawk is a default member, so plain `cargo build` / `cargo publish` +# act on sqawk alone; use `--workspace` (as CI does) to include tsq. +[workspace] +members = ["tsq"] +default-members = ["."] diff --git a/README.md b/README.md index c024b35..398cf17 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,6 @@ # Sqawk [![Crates.io](https://img.shields.io/crates/v/sqawk.svg)](https://crates.io/crates/sqawk) -[![Docs.rs](https://docs.rs/sqawk/badge.svg)](https://docs.rs/sqawk) [![MIT licensed](https://img.shields.io/crates/l/sqawk.svg)](./LICENSE) Sqawk is an SQL-based command-line tool for processing delimiter-separated files (CSV, TSV, etc.), inspired by the classic `awk` command. It loads data into in-memory tables, executes SQL queries, and optionally writes results back to files. @@ -10,14 +9,15 @@ Sqawk is an SQL-based command-line tool for processing delimiter-separated files - **SQL Query Engine** - SELECT, INSERT, UPDATE, DELETE with WHERE, ORDER BY, GROUP BY, HAVING, LIMIT/OFFSET - **Joins** - INNER, LEFT, RIGHT, FULL OUTER, and CROSS joins with ON conditions -- **Aggregates** - COUNT, SUM, AVG, MIN, MAX with GROUP BY support -- **Functions** - String (UPPER, LOWER, SUBSTR, REPLACE, etc.), math (ABS, ROUND, etc.), date/time +- **Aggregates** - COUNT, SUM, TOTAL, AVG, MIN, MAX, GROUP_CONCAT/STRING_AGG with GROUP BY, DISTINCT and `FILTER (WHERE ...)` +- **Functions** - String (UPPER, LOWER, LENGTH, SUBSTR, INSTR, REPLACE, TRIM, LPAD/RPAD, PRINTF, ...), math (ABS, ROUND, CEIL, FLOOR, SIGN, multi-argument MIN/MAX), conditional (COALESCE, IFNULL, NULLIF, IIF, TYPEOF), date/time (DATE, TIME, DATETIME, STRFTIME, JULIANDAY, UNIXEPOCH with modifiers) +- **SQLite semantics** - 64-bit integer arithmetic, SQLite CAST rules, ASCII case-insensitive LIKE with ESCAPE, character-based strings; unimplemented syntax is an error, never silently ignored - **Subqueries** - Scalar, `IN (SELECT ...)`, and `EXISTS`, including correlated - **Set Operations** - UNION, UNION ALL, INTERSECT, EXCEPT -- **Window Functions** - ROW_NUMBER, RANK, DENSE_RANK, LAG, LEAD, and aggregates with `OVER (PARTITION BY ... ORDER BY ...)` +- **Window Functions** - ROW_NUMBER, RANK, DENSE_RANK, PERCENT_RANK, CUME_DIST, NTILE, LAG, LEAD, FIRST_VALUE, LAST_VALUE, NTH_VALUE, and aggregates with `OVER (PARTITION BY ... ORDER BY ...)`, ROWS frames and named windows - **DDL** - CREATE TABLE, CREATE TABLE AS SELECT, DROP, ALTER TABLE ADD COLUMN, TRUNCATE -- **Expressions** - CASE, CAST, COALESCE, NULLIF, BETWEEN, IN, LIKE/ILIKE, `||`, arithmetic -- **File Formats** - CSV, TSV, and custom delimiters; headerless files via `--tabledef` +- **Expressions** - CASE, CAST, BETWEEN, IN, LIKE/ILIKE ... ESCAPE, IS [NOT] DISTINCT FROM, `||`, arithmetic; ORDER BY ... NULLS FIRST/LAST +- **File Formats** - CSV, TSV, and custom delimiters; headerless files via `--no-header` or `--tabledef`; unchanged cells survive `--write` byte for byte - **Pipelines** - Reads standard input with a `-` file operand - **Safe by Default** - Files unchanged unless `--write` flag is specified - **Interactive REPL** - Explore data interactively with `-i` flag @@ -28,8 +28,7 @@ Sqawk is an SQL-based command-line tool for processing delimiter-separated files cargo install sqawk ``` -This installs two binaries: `sqawk` and [`tsq`](#tsq---test-data-generator). -Requires Rust 1.88 or newer. +This installs the `sqawk` binary. Requires Rust 1.88 or newer. ## Quick Examples @@ -50,13 +49,15 @@ sqawk -s "UPDATE data SET status = 'archived' WHERE year < 2020" data.csv --writ cat employees.csv | sqawk -s "SELECT name FROM stdin WHERE salary > 70000" - ``` -## `tsq` - test data generator +## `tsq` - test data generator (development only) -`cargo install sqawk` also installs `tsq`, which generates deterministic -multi-table CSV data plus a corpus of SQL queries for exercising sqawk. +The source tree also contains `tsq`, which generates deterministic multi-table +CSV data plus a corpus of SQL queries for exercising sqawk. It is a separate +workspace crate (`tsq/`) that is not published, so `cargo install sqawk` does +not install it. Run it from a checkout: ```sh -tsq --seed 42 --rows 1000 --output-dir /tmp/sqawk-test +cargo run -p tsq -- --seed 42 --rows 1000 --output-dir /tmp/sqawk-test sqawk -s "SELECT * FROM customers LIMIT 10" /tmp/sqawk-test/data/customers.csv ``` diff --git a/doc/database.md b/doc/database.md index 9f3c2f5..1c932c2 100644 --- a/doc/database.md +++ b/doc/database.md @@ -83,11 +83,81 @@ writes back only tables that actually changed. The compiler has one recursive expression compiler, `code_expr` (the analogue of SQLite's `sqlite3ExprCode`), used from every position an expression can appear: the SELECT list, WHERE, HAVING, ORDER BY, SET clauses, join conditions, -function arguments and correlated subqueries. Name resolution is carried -separately in a `NameCtx`, which models the visible FROM sources, whether a -column lives behind a cursor or in an already-loaded register, and the -inner/outer scopes a correlated subquery needs. A node supported in one -position is therefore supported in all of them. +function arguments and subqueries. Name resolution is carried separately in a +`NameCtx`, which models the visible FROM sources, whether a column lives +behind a cursor or in an already-loaded register, and -- for a subquery -- a +link to the enclosing query's scope. A node supported in one position is +therefore supported in all of them. + +### SELECT Pipeline + +Every SELECT core runs one pipeline (`src/vm/compiler_select.rs`): + + FROM planner -> WHERE -> [aggregate stage -> HAVING] -> [window stage] -> output stage + +- A SELECT without FROM is a FROM plan with zero sources, producing one row, + so WHERE, ORDER BY, DISTINCT and LIMIT apply to it as to any other query. +- The aggregate stage collects aggregate calls from the SELECT list, HAVING + and ORDER BY. Without GROUP BY there is always exactly one output row (COUNT + of no rows is 0, SUM/AVG/MIN/MAX of no rows NULL); with GROUP BY, no input + rows means no output rows. +- The window stage runs after aggregation, so a window may range over + grouped rows (`SUM(COUNT(*)) OVER ()`) and over any join. Each finished row + goes into an ephemeral table with everything the output needs (all FROM + columns, or the GROUP BY keys and aggregates) plus each window's partition + keys, order keys and arguments. For each distinct window definition + (PARTITION BY + ORDER BY) one `WindowCompute` stably sorts the table by its + keys, walks the partitions and peer groups, and writes each function's + value -- each with its own state and frame -- into a result column. The + definitions are computed last-first, so without ORDER BY rows come out in + the first window's order, as in SQLite. The output stage then reads the + rows back, with every window call bound to its value like an aggregate, so + a window call may appear in any expression and in ORDER BY; anywhere else + (WHERE, GROUP BY, HAVING, an aggregate argument) it is an error. +- The output stage computes the output columns, then ORDER BY (an ordinal, a + SELECT alias, a projected expression, or else a hidden sort key such as an + unprojected column or aggregate), DISTINCT, and LIMIT/OFFSET. HAVING and + sort-key expressions may name SELECT aliases. LIMIT and OFFSET accept any + expression yielding an integer; a negative LIMIT means no limit. +- Compound SELECTs (UNION [ALL], INTERSECT, EXCEPT, in any chain) take their + header from the leftmost SELECT; ORDER BY (ordinals or leftmost column + names) and LIMIT/OFFSET apply to the whole compound, and a DISTINCT on one + arm applies to that arm only. As in SQLite, set operators use keyed + ephemeral tables (`OpenEphemeral` with p3 = 1, `IdxInsert`, `IdxDelete`, + `Found`/`NotFound`), and without ORDER BY a UNION, INTERSECT or EXCEPT + result comes out sorted. INTERSECT ALL / EXCEPT ALL are not supported. + +### Subqueries + +A subquery (scalar, `EXISTS`, `IN`) is compiled by the same query compiler as +a statement, as a fragment of the enclosing statement's program placed where +the expression is evaluated (`src/vm/compiler_subquery.rs`), as in SQLite. So +every SELECT feature works inside one. Only the destination differs: a scalar +subquery copies its first row's value into a register and stops, EXISTS sets +a flag on its first row and stops, and IN inserts its rows into a keyed +ephemeral table that the left operand probes with `Found`, with SQL's +three-valued result (a NULL member makes a miss NULL; an empty set makes IN +false even for a NULL operand). + +- **Name resolution**: the subquery's FROM scope links to the scope of the + expression containing it. A name is resolved innermost scope first, then + outward, so the inner table shadows an outer one with the same column and an + outer table is found by its alias or, unaliased, by its own name even when + the subquery reads the same table under another alias. +- **Correlation**: a subquery that resolves any name outward -- itself or + through a subquery nested in it -- is correlated. This is decided during + name resolution, not by matching qualifier strings. An uncorrelated + subquery's fragment is guarded by `Once` and runs for the first evaluation + only; a correlated one reruns on every evaluation, i.e. per outer row. +- **Correlated execution**: a subquery's cursors and hash indexes do not + depend on the outer row, so they are set up once (`Once` around the + `OpenRead`s and hash builds). An equality between an inner column and an + outer column -- `WHERE d.k = f.k` -- becomes a hash probe, so a correlated + `EXISTS`/`IN`/scalar lookup costs one probe per outer row instead of a + rescan. Aggregate accumulators, sorters and DISTINCT tables are reset at + the start of every run. +- A subquery's cursors are numbered apart from the statement's, so it never + disturbs an enclosing scan. ### Statement Execution Model @@ -107,23 +177,16 @@ lasts only for that statement. ### File Handlers -The file handling system consists of multiple components: - -#### `FileHandler` Trait -- Defines a common interface for different file format handlers -- Provides abstraction for loading and saving tables from various file formats -- Allows for consistent treatment of different delimiter-separated formats - -#### `CsvHandler` Implementation -- Specialized handler for standard CSV files -- Uses the csv crate for parsing and writing -- Handles comma-separated values with standard CSV escaping rules - -#### `DelimHandler` Implementation -- Handles files with custom delimiters (e.g., TSV, colon-separated, etc.) -- Configured via the `-F` command-line option -- Supports tab, colon, pipe, and other custom separators -- Preserves the original delimiter and format during writeback +- `FileHandler` (`file_handler.rs`) resolves `[table_name=]path` operands, + chooses the delimiter (`-F`, else comma for `.csv`, no extension and stdin, + tab otherwise) and the header policy (`.csv` and stdin: header always; + other files: headerless guess and `#` comments; `--no-header`/`--tabledef`: + no header), and routes writeback. +- `csv_handler.rs` does all parsing and writing. Every format is read by the + `csv` crate with one configuration (RFC 4180 quoting, multi-line quoted + fields, no trimming, BOM stripped, ragged rows padded), and written back by + a small RFC 4180 record writer that also quotes fields with leading or + trailing whitespace. ## File I/O Patterns @@ -132,8 +195,9 @@ The file handling system consists of multiple components: When saving modified tables back to disk (via the `--write` flag), Sqawk uses an atomic write pattern to ensure data integrity: 1. **Write to Temporary File**: Data is first written to a temporary file in the same directory as the target file -2. **Sync to Disk**: The temporary file is explicitly synced (`fsync`) to ensure data is durably stored on the physical device -3. **Atomic Rename**: The temporary file is renamed to the target filename using an atomic rename operation +2. **Keep Permissions**: The original file's permissions are copied onto the temporary file +3. **Sync to Disk**: The temporary file is explicitly synced (`fsync`) to ensure data is durably stored on the physical device +4. **Atomic Rename**: The temporary file is renamed to the target filename using an atomic rename operation This pattern provides several safety guarantees: @@ -141,14 +205,13 @@ This pattern provides several safety guarantees: - **Disk Full Protection**: If the disk runs out of space, the original file is preserved - **All-or-Nothing Semantics**: Either the complete new file exists, or the original file is unchanged -The temporary file is created in the same directory as the target file to ensure the rename operation is atomic (same filesystem requirement on POSIX systems). +The temporary file is created in the same directory as the target file to ensure the rename operation is atomic (same filesystem requirement on POSIX systems). A symlinked path is resolved first, so the link's target is rewritten and the link itself is left in place. ### Delimiter Consistency Sqawk maintains delimiter consistency between input and output: - When a file is loaded with a specific delimiter (via `-F` flag or file extension detection), that delimiter is preserved in the table metadata -- Query results are output using the same delimiter as the source table - When files are written back, the original delimiter format is preserved This ensures that TSV files remain TSV, colon-separated files remain colon-separated, and so on. The principle of least surprise: output format matches input format unless explicitly overridden. @@ -164,76 +227,71 @@ Sqawk supports two primary output modes: For stdout output, the data is streamed directly without buffering concerns. For file writeback, the atomic write pattern described above is used. -## Storage Backend Architecture +## Storage + +Every table holds its rows in memory as owned values (`Vec`). Files are +read whole and parsed by the `csv` crate -- RFC 4180 quoting, quoted fields +spanning lines, no trimming, UTF-8 byte order mark stripped -- and each cell's +type is inferred with a rule that round-trips (see "Type inference" below), so +an unmodified cell is written back byte for byte. -Sqawk supports multiple storage backends for table row data, enabling efficient handling of different data sources: +Standard input (the `-` operand) is read the same way, directly from the pipe. +The resulting table has no file path, so `--write` reports it as unwritable +rather than writing anywhere. -### Memory Storage +### Type inference -The `MemoryStorage` backend stores all row data in heap-allocated memory: -- Used for tables with no backing file (`CREATE TABLE`, `CREATE TABLE AS - SELECT`), and for any mmap table after its first modification -- All string values are owned (heap-allocated) -- Supports all operations including INSERT, UPDATE, DELETE -- Data is mutable throughout the session +- empty field: NULL +- canonical integer (`42`, `-7`, where formatting the number back gives the + same text): INTEGER +- canonical float (`1.5`, `-0.25`): REAL +- anything else, verbatim: TEXT -- including `007`, `1.50`, `1e3`, ` 42`, + `true`, `NaN` -### Mmap Storage (Default for on-disk files) +Text that spells a number still acts as one where a number is needed: +arithmetic, `SUM`/`AVG`, comparison against a number, and sort order. -The `MmapStorage` backend uses memory-mapped files for zero-copy access: -- Automatically used when loading files from disk -- String values borrow directly from the mmap'd region (zero allocation) -- Read-only by default to maintain zero-copy efficiency -- Converts to MemoryStorage on first write operation +### Source layout -### Automatic Backend Selection +Each table records how its file was laid out -- whether it had a header line, +whether records ended in CRLF, and whether `#` lines were comments -- and +writeback reproduces it. Writeback also keeps the file's permissions and, for +a symlink, rewrites the link's target and leaves the link in place. -The storage backend is selected automatically based on the data source: -- **On-disk files**: Uses mmap for zero-copy loading -- **Tables created by DDL**: Uses in-memory storage; there is no file to map -- **Write operations**: Mmap tables convert to memory on first modification +In a file read with comment skipping (any delimited file that is not `.csv`), +each `#` comment line is kept verbatim, with its own line ending, and +anchored to the data that follows it: comments before the header (or, in a +headerless file, before the first row) stay at the top, comments before a +row stay with that row, and comments after the last row stay at the end. So +a no-op UPDATE writes such a file back byte for byte. When rows change: -Standard input (the `-` operand) is not seekable and cannot be mapped directly, -so it is spooled to a temporary file and then loaded by the ordinary mmap path. -The temp file handle is held for the run's duration, because the mmap borrows -from it. The resulting table is then detached from that path, so `--write` has -no target and reports the table as unwritable rather than rewriting a temp file -that is about to be deleted. +- UPDATE replaces rows in place, so every comment stays where it was. +- DELETE leaves a deleted row's comments where they were: in front of the + next surviving row, or of the trailing comments. Deleting every row (or + TRUNCATE) keeps all comments, in order. +- INSERT appends rows after the last row and before the trailing comments. + +Blank lines are not kept (the reader skips them), and `#` lines in a `.csv` +file are data, not comments. ### Row Identity Across Updates UPDATE replaces a row in place rather than deleting and re-appending it, so a table keeps its row order and `--write` does not reorder the user's file. -### Write Operation Behavior - -When a table with mmap storage is modified (INSERT, UPDATE, DELETE): -1. All rows are copied to heap memory (converting Cow::Borrowed to Cow::Owned) -2. The storage backend switches from Mmap to Memory -3. Subsequent operations use the in-memory copy -4. The original file remains unchanged until `--write` is specified - -This design provides: -- Zero-allocation loading for read-only queries -- Transparent handling of write operations -- Memory efficiency for large read-only datasets - ## Performance Considerations Sqawk's in-memory database is optimized for: - **Fast Loading**: Delimiter-separated files are parsed directly into memory -- **Zero-Copy Access**: Memory-mapped files enable direct access without allocation overhead - **Format Flexibility**: Support for CSV, TSV, and custom-delimited files - **Efficient Filtering**: WHERE clauses are applied in a single pass - **Low Memory Overhead**: Simple data structures minimize memory usage - **Zero Configuration**: No setup required, works directly with files in various formats For larger datasets, consider: -- The entire dataset must fit in memory (or be addressable via mmap) +- The entire dataset must fit in memory - Complex queries may require multiple passes over the data -- Write operations on mmap tables trigger a copy to memory -- Custom delimiters may have slightly different performance characteristics than standard CSV -- Read-only operations on large files benefit most from mmap storage ## Join Implementation @@ -252,35 +310,45 @@ The database engine supports combining data from multiple tables through SQL-sta - **ON constraints**: Supported, and the condition is a full expression, not just a single comparison: `ON a.id = b.id AND UPPER(a.name) = 'X'` - **Table aliases**: Supported (`FROM users u JOIN orders o ON u.id = o.user_id`) -- **USING / NATURAL**: Not supported -- **Multi-table joins**: Supports chaining 3+ tables in a single query -- **Aggregates over joins**: Supported for inner joins, which are rewritten to - the equivalent comma join. Deliberately NOT applied to outer joins, since - moving the ON condition into WHERE would discard the NULL-extended rows an - outer join exists to produce +- **USING / NATURAL**: Supported; the shared columns are merged into one + output column (the first non-NULL of the joined columns) +- **Multi-table joins**: Any chain of INNER / LEFT / CROSS joins and comma + lists. RIGHT JOIN is supported when everything to its left is inner-joined; + FULL JOIN must be the last join +- **Expressions**: The SELECT list, ON, WHERE, GROUP BY, HAVING and ORDER BY + of a join accept the same expressions as a single-table query +- **Aggregates over joins**: Supported for every join type, including the + NULL-extended rows of outer joins ### Technical Implementation -- Joins are compiled to VM bytecode for efficient execution -- Nested loop algorithm with appropriate NULL handling for outer joins -- Column references use table qualifiers for disambiguation in result sets -- Type coercion rules are applied consistently in join conditions +- One FROM planner (`src/vm/compiler_from.rs`) serves every SELECT: each + source table is a cursor and a loop level, outermost first +- ON conditions are evaluated at their join's level; WHERE conditions after + outer-join NULL-extension, at the innermost level that has all their tables +- Outer joins keep a per-level match flag; an unmatched row re-runs the inner + levels once with the inner tables on a NULL row (`NullRow`). RIGHT JOIN is a + LEFT JOIN with the loop order swapped; FULL JOIN adds a pass over the right + table for rows that never matched +- An equality between the inner table and the outer tables becomes a hash + join (`HashInsert` / `HashProbe` / `HashNext`). Keys are normalized so that + values `=` treats as equal (1, 1.0, '1') share a bucket; NULL keys never + match; the full join condition is re-checked on every candidate ## Current System Limitations The database engine has several architectural limitations: -- **No Index Structure**: All operations perform full table scans +- **No Index Structure**: Tables are scanned in full; the only index is the + temporary hash index an equi-join builds for its inner table - **No Transaction Support**: A statement's modifications are applied once it completes; there is no BEGIN/COMMIT and no rollback across statements - **Schema Flexibility**: Types are inferred per value rather than enforced - **No Constraints System**: Referential integrity not enforced -- **Whole-dataset residency**: A table must fit in memory, or be addressable - via mmap +- **Whole-dataset residency**: A table must fit in memory -Unsupported SQL of note: CTEs (`WITH`), explicit window frames -(`ROWS BETWEEN ...`), `USING`/`NATURAL` joins, and aggregates over outer -joins. +Unsupported SQL of note: CTEs (`WITH`), parenthesized joins, and RANGE +frames with an offset, GROUPS frames and FILTER on window functions. --- diff --git a/doc/sql_reference.md b/doc/sql_reference.md index ee21577..14556c2 100644 --- a/doc/sql_reference.md +++ b/doc/sql_reference.md @@ -10,7 +10,12 @@ | `BOOLEAN` | `BOOL` | | `NULL` | | -Type inference on load: Integer → Float → Boolean → String. Empty values become NULL. +Type inference on load, per cell: an empty field is NULL; a canonical integer +(`42`, `-7` -- printing the number gives back the same text) is INTEGER; a +canonical float (`1.5`, `-0.25`) is REAL; everything else is TEXT, verbatim and +untrimmed -- including `007`, `1.50`, `1e3`, ` 42`, `true` and `NaN`. Booleans +are never inferred. The rule guarantees that every loaded cell prints, and is +written back, exactly as it was read. ## SELECT @@ -21,7 +26,7 @@ FROM table [alias] [, table2 ...] [WHERE condition] [GROUP BY columns] [HAVING condition] -[ORDER BY column [ASC|DESC], ...] +[ORDER BY expr [ASC|DESC] [NULLS FIRST|NULLS LAST], ...] [LIMIT n [OFFSET m]] ``` @@ -43,11 +48,19 @@ SELECT col alias -- alias without AS | `<`, `>`, `<=`, `>=` | `col > 10` | | `AND`, `OR`, `NOT` | `a > 1 AND b < 5` | | `IS NULL`, `IS NOT NULL` | `col IS NULL` | -| `LIKE`, `ILIKE` | `name LIKE 'A%'` | +| `LIKE`, `ILIKE` | `name LIKE 'A%'`, `code LIKE 'x\_%' ESCAPE '\'` | +| `IS [NOT] DISTINCT FROM` | `a IS NOT DISTINCT FROM b` (NULL-safe `=`) | | `BETWEEN` | `col BETWEEN 1 AND 10` | | `IN` | `col IN (1, 2, 3)` | | `IN (SELECT ...)` | `id IN (SELECT id FROM t)` | +`LIKE` follows SQLite: `%` matches any run of characters and `_` exactly one +character (not byte), both including newlines; letters compare +case-insensitively for ASCII only (`'a%'` matches `Alice`, but `'é'` does not +match `'É'`). `ESCAPE 'c'` makes the character after `c` literal. The pattern +may be any expression, such as another column. `ILIKE` (not in SQLite) folds +Unicode case. A NULL value, pattern or escape gives NULL. + ### CASE Expression ```sql @@ -60,58 +73,126 @@ CASE expr WHEN val THEN result [WHEN ...] [ELSE default] END | Function | Description | |----------|-------------| | `COUNT(*)`, `COUNT(col)`, `COUNT(DISTINCT col)` | Row/value count | -| `SUM(col)` | Sum of values | -| `AVG(col)` | Average | -| `MIN(col)` | Minimum | -| `MAX(col)` | Maximum | +| `SUM(x)` | Sum; an integer while every value is an integer | +| `TOTAL(x)` | Sum as a real; `0.0` (not NULL) over no rows | +| `AVG(x)` | Average (a real) | +| `MIN(x)`, `MAX(x)` | Least / greatest value | +| `GROUP_CONCAT(x [, sep])` | Non-NULL values joined by `sep` (default `,`) | +| `STRING_AGG(x, sep)` | Same as `GROUP_CONCAT(x, sep)` | + +Any aggregate takes `DISTINCT` (`SUM(DISTINCT x)`; `GROUP_CONCAT(DISTINCT x)` +only with one argument) and a filter: `COUNT(*) FILTER (WHERE age > 30)` +aggregates only the rows for which the condition is true. Aggregates skip +NULLs; over no rows COUNT is 0, TOTAL 0.0 and the rest NULL. SUM of integers +that overflows 64 bits is an `integer overflow` error, as in SQLite (TOTAL and +AVG carry on in floating point). Text that spells a number is summed as that +number; other text is skipped. ### String Functions +String functions count characters, not bytes. Unless noted, a NULL argument +gives NULL, and a number argument is used in its text form. + | Function | Description | |----------|-------------| -| `UPPER(s)` | Uppercase | -| `LOWER(s)` | Lowercase | -| `TRIM(s)` | Remove leading/trailing whitespace | -| `SUBSTR(s, start [, len])` | Substring (1-indexed) | +| `UPPER(s)`, `LOWER(s)` | Case conversion (all of Unicode) | +| `LENGTH(s)` | Number of characters | +| `TRIM(s [, chars])`, `LTRIM`, `RTRIM` | Strip whitespace, or any of `chars`, from both / the start / the end | +| `TRIM([BOTH\|LEADING\|TRAILING] chars FROM s)` | Standard-SQL spelling of the same | +| `SUBSTR(s, start [, len])`, `SUBSTRING` | Substring (see below) | | `SUBSTRING(s FROM start [FOR len])` | Substring (alternate syntax) | -| `REPLACE(s, from, to)` | Replace occurrences | -| `CONCAT(s1, s2, ...)` | Concatenate strings | -| `LENGTH(s)` | String length | -| `LEFT(s, n)` | First n characters | -| `RIGHT(s, n)` | Last n characters | -| `LPAD(s, len, pad)` | Left-pad to length | -| `RPAD(s, len, pad)` | Right-pad to length | +| `REPLACE(s, from, to)` | Replace every occurrence; an empty `from` leaves `s` alone | +| `INSTR(s, t)` | 1-based position of the first `t` in `s`, 0 if absent | +| `CONCAT(a, b, ...)` | Concatenation; NULL arguments count as `''` | +| `CONCAT_WS(sep, a, ...)` | Concatenation with a separator; NULL arguments skipped | +| `LEFT(s, n)`, `RIGHT(s, n)` | First / last `n` characters | +| `LPAD(s, len [, fill])`, `RPAD` | Pad to `len` characters (see below) | +| `HEX(x)` | Upper-case hex of the UTF-8 bytes; `''` for NULL | +| `CHAR(n, ...)` | The characters with these code points | +| `UNICODE(s)` | Code point of the first character | +| `PRINTF(fmt, ...)`, `FORMAT` | C-style formatting: `%d %i %u %f %e %g %x %X %o %s %c %%`, flags `-+ 0`, width, precision | +| `a \|\| b` | Concatenation; NULL if either side is NULL | + +`SUBSTR` follows SQLite: positions are 1-based; a negative `start` counts +from the end (`SUBSTR('hello', -3)` is `llo`); `start` 0 is a position before +the first character, so `SUBSTR('hello', 0, 2)` is `h`; a negative `len` takes +the characters before `start` (`SUBSTR('hello', 3, -2)` is `he`). + +`LPAD`/`RPAD` are not in SQLite and follow PostgreSQL: the fill (default a +space) is repeated as needed, `LPAD('ab', 5, 'xy')` is `xyxab`; a string +longer than `len` is cut to its first `len` characters (`LPAD('hello', 2)` is +`he`); an empty fill leaves a short string as it is; `len <= 0` gives `''`. + +No function builds a string longer than SQLite's default length limit, +1,000,000,000 bytes: `LPAD('a', 10000000000)`, a `PRINTF` width or precision +past the limit, or a `REPLACE`, `HEX` or concatenation whose result would +exceed it is the error "string or blob too big". The length is checked before +anything is allocated. ### Math Functions | Function | Description | |----------|-------------| | `ABS(n)` | Absolute value | -| `ROUND(n [, d])` | Round to `d` decimal places, or to the nearest integer | +| `ROUND(n [, d])` | Round to `d` decimal places (default 0); always a real | | `CEIL(n)`, `CEILING(n)` | Round up | | `FLOOR(n)` | Round down | +| `SIGN(n)` | -1, 0 or 1; NULL for text that is not a number | +| `MIN(a, b, ...)`, `MAX(a, b, ...)` | Least / greatest argument; NULL if any is NULL | + +`ROUND(3.14159, 2)` is `3.14`. Halves round away from zero, judged on the +value actually stored: `ROUND(2.675, 2)` is `2.67`, because 2.675 is stored as +2.67499999.... A negative `d` rounds to the left of the decimal point, so +`ROUND(1234.5, -2)` is `1200` (SQLite treats a negative `d` as 0). A `d` that +is not a number is an error rather than a silent `0`. -`ROUND(3.14159, 2)` is `3.14`. A negative `d` rounds to the left of the decimal -point, so `ROUND(1234.5, -2)` is `1200`. A `d` that is not a number is an error -rather than a silent `0`. `ABS`, `CEIL`, `CEILING` and `FLOOR` take exactly one -argument and reject a second. +`ABS` of an integer is an integer, and `ABS(-9223372036854775808)` is an +`integer overflow` error. `CEIL` and `FLOOR` return an integer argument +unchanged. With one argument, `MIN` and `MAX` are the aggregates. -A result that lands on a whole number prints without a decimal part, as -everywhere else in sqawk: `ROUND(2.0, 2)` is `2`, matching `AVG` and plain -arithmetic rather than SQL's `2.0`. +A real that lands on a whole number prints without a decimal part, as +everywhere else in sqawk: `ROUND(2.0, 2)` is `2`, where SQLite prints `2.0`. ### Arithmetic -`+`, `-`, `*`, `/`, `%` (modulo) +`+`, `-`, `*`, `/`, `%` follow SQLite: + +- Integers stay 64-bit integers: `5 / 2` is `2`, `-7 / 2` is `-3` (truncation + toward zero), `7 % 3` is `1`. Integers beyond 2^53 stay exact. +- A result that does not fit in 64 bits is computed as a real instead: + `9223372036854775807 + 1` is `9223372036854775808.0`. +- A real operand makes the result real (`7.0 / 2` is `3.5`). `%` with a real + operand takes the remainder of the operands truncated to integers, as a + real (`5.5 % 2` is `1.0`). +- Division or remainder by zero is NULL. Any NULL operand gives NULL. +- Text is read by its longest numeric prefix: `'12abc' + 0` is `12`, + `' 5 ' + 1` is `6`, `'1e3' + 0` is the real `1000`, `'abc' + 1` is `1`. ### Conditional and Conversion | Expression | Description | |------------|-------------| -| `CAST(expr AS TYPE)` | Convert to `INTEGER`, `REAL`, `TEXT`, `BOOLEAN` | +| `CAST(expr AS type)` | Convert (see below) | | `COALESCE(a, b, ...)` | First non-NULL argument | +| `IFNULL(a, b)` | `COALESCE(a, b)` | | `NULLIF(a, b)` | NULL when `a = b`, otherwise `a` | -| `a \|\| b` | String concatenation | +| `IIF(cond, a, b)` | `CASE WHEN cond THEN a ELSE b END` | +| `TYPEOF(x)` | `'integer'`, `'real'`, `'text'` or `'null'` | + +COALESCE, IFNULL and IIF evaluate only the argument they return. + +`CAST` maps any type name onto INTEGER, REAL, TEXT or NUMERIC by SQLite's +affinity rules: a name containing `INT` is INTEGER (`BIGINT`, `SMALLINT`); +`CHAR`, `CLOB` or `TEXT` is TEXT (`VARCHAR(10)`); `REAL`, `FLOA` or `DOUB` is +REAL; anything else (`NUMERIC`, `DECIMAL`, `BOOLEAN`, `DATE`) is NUMERIC. + +- `CAST(text AS INTEGER)` reads an optional sign and the digits after it: + `'3.9'` is 3, `'12abc'` is 12, `' 5 '` is 5, `'1e3'` is 1, `'abc'` is 0. + Out-of-range values saturate at the 64-bit limits. +- `CAST(real AS INTEGER)` truncates toward zero, saturating. +- `CAST(text AS REAL)` reads the longest numeric prefix (`'1e3x'` is 1000). +- `CAST(text AS NUMERIC)` gives an integer when the number is whole + (`'3.0'` is 3), else a real. `CAST(1 AS BOOLEAN)` is 1. ### Result Column Names @@ -125,11 +206,33 @@ addressable. Use `AS` whenever the header is consumed downstream. | Function | Description | |----------|-------------| -| `NOW()`, `CURRENT_TIMESTAMP` | Current timestamp | -| `CURRENT_DATE` | Current date | -| `CURRENT_TIME` | Current time | -| `DATE(expr)` | Extract date | -| `TIME(expr)` | Extract time | +| `DATE(t, modifier...)` | `YYYY-MM-DD` | +| `TIME(t, modifier...)` | `HH:MM:SS` | +| `DATETIME(t, modifier...)` | `YYYY-MM-DD HH:MM:SS` | +| `JULIANDAY(t, modifier...)` | Julian day number (a real) | +| `UNIXEPOCH(t, modifier...)` | Seconds since 1970-01-01 (an integer) | +| `STRFTIME(fmt, t, modifier...)` | Formatted: `%d %f %H %j %J %m %M %s %S %w %W %Y %%` | +| `CURRENT_DATE`, `CURRENT_TIME`, `CURRENT_TIMESTAMP`, `NOW()` | Now | + +These follow SQLite, in UTC. The time string `t` is `now`, `YYYY-MM-DD`, +`YYYY-MM-DD HH:MM[:SS[.fff]]` (or with `T` for the space, and an optional +`Z` / `+HH:MM` zone), `HH:MM[:SS]` (on 2000-01-01), or a number: a Julian day +number, or seconds since 1970 when the first modifier is `unixepoch`. With no +arguments the functions return the current moment. + +Modifiers: `+N days|hours|minutes|seconds|months|years` (or `-N`; fractions +allowed), `start of day|month|year`, +`weekday N` (0 = Sunday), `unixepoch`, `utc`. `localtime` is an error, since +sqawk has no time zone. + +A malformed time string or an unknown modifier gives NULL. A day past the end +of its month, up to 31, rolls into the next month: `DATE('2024-02-30')` is +`2024-03-01`, and `DATE('2024-01-31', '+1 month')` is `2024-03-02`. +As in SQLite, a fractional month or year count moves the calendar by its +whole part and then adds the fraction as 30-day months or 365-day years +(`DATETIME('2024-01-01', '+1.5 months')` is `2024-02-16 00:00:00`); a count +beyond a unit's range (about 14713 years) gives NULL; and the result must lie +in years 0000-9999, which is checked after the last modifier. ## JOIN @@ -145,12 +248,27 @@ SELECT * FROM t1 LEFT JOIN t2 ON t1.id = t2.id SELECT * FROM t1 RIGHT JOIN t2 ON t1.id = t2.id SELECT * FROM t1 FULL JOIN t2 ON t1.id = t2.id --- Multiple joins +-- Multiple joins, any mix of INNER / LEFT / CROSS SELECT * FROM t1 JOIN t2 ON t1.id = t2.t1_id - JOIN t3 ON t2.id = t3.t2_id + LEFT JOIN t3 ON t2.id = t3.t2_id + +-- USING and NATURAL: the shared columns appear once in SELECT * +SELECT * FROM t1 JOIN t2 USING (id) +SELECT * FROM t1 NATURAL LEFT JOIN t2 ``` +WHERE filters the joined rows, including the NULL-extended rows of an outer +join (`... LEFT JOIN t2 ON ... WHERE t2.id IS NULL` is an anti-join). An +unqualified column name must be unambiguous across the joined tables. + +A RIGHT JOIN whose left side already contains an outer join, and a FULL JOIN +followed by further joins, are rejected as unsupported. + +Joins on an equality (`ON a.k = b.k`, or the same condition in WHERE for a +comma or inner join) run as hash joins: the inner table is indexed once +instead of being rescanned for every outer row. + ## Subqueries ```sql @@ -170,33 +288,81 @@ SELECT name FROM employees e ``` ```sql --- Derived table: a subquery in FROM, which must be given an alias +-- Derived table: a subquery in FROM SELECT name FROM (SELECT name, salary FROM employees WHERE salary > 70000) t SELECT COUNT(*) FROM (SELECT department FROM employees GROUP BY department) t ``` -A derived table is materialized before the outer query runs and exists only for -the statement that declares it. Its alias may not shadow a real table. +A derived table is materialized before the statement runs and exists only for +the statement that declares it. It may appear in any FROM -- including inside +a subquery, a compound arm or an INSERT ... SELECT -- and its alias is +optional and may shadow a real table's name. A derived table that refers to a +column of an enclosing query cannot be materialized up front and is an error. + +A subquery is a full query: it may use joins, GROUP BY, HAVING, DISTINCT, +ORDER BY, LIMIT/OFFSET and UNION/INTERSECT/EXCEPT, and it may appear anywhere +an expression can -- the SELECT list, WHERE, ON, HAVING, ORDER BY, LIMIT, CASE, +function and aggregate arguments, UPDATE SET, DELETE WHERE and INSERT VALUES. + +- A name the subquery does not define is looked up in the enclosing queries, + innermost first, so a subquery may reference a column of any enclosing + query, at any depth. Such a subquery is *correlated* and runs again for each + outer row; an uncorrelated one runs once. +- A scalar subquery yields its first row's value, or NULL when it returns no + rows. It must return exactly one column (*"sub-select returns 2 columns - + expected 1"*); so must the subquery of IN. +- `x IN (SELECT ...)` follows SQL's three-valued logic: over an empty result + it is false (NOT IN true) even for a NULL `x`; otherwise a NULL `x`, or a + miss when the result contains a NULL, is NULL -- so `NOT IN` over a result + containing a NULL is never true. +- In UPDATE and DELETE, a subquery sees the table as it was before the + statement started. (SQLite instead lets a correlated subquery see rows the + same UPDATE has already changed.) +- Not supported: row values (`(a, b) IN (SELECT x, y ...)`), `ANY`/`ALL`. ## Window Functions ```sql SELECT name, ROW_NUMBER() OVER (ORDER BY salary DESC) FROM employees -SELECT name, RANK() OVER (ORDER BY department) FROM employees -SELECT name, DENSE_RANK() OVER (ORDER BY department) FROM employees -SELECT name, LAG(salary) OVER (ORDER BY salary) FROM employees -SELECT name, LEAD(salary) OVER (ORDER BY salary) FROM employees - --- Aggregates over a window -SELECT name, SUM(salary) OVER (PARTITION BY department) FROM employees +SELECT name, RANK() OVER (PARTITION BY department ORDER BY salary) FROM employees +SELECT name, LAG(salary, 1, 0) OVER (ORDER BY salary) FROM employees +SELECT name, salary - AVG(salary) OVER (PARTITION BY department) FROM employees +SELECT department, COUNT(*), RANK() OVER (ORDER BY COUNT(*) DESC) + FROM employees GROUP BY department +SELECT name, SUM(salary) OVER (ORDER BY id ROWS BETWEEN 1 PRECEDING AND 1 FOLLOWING) + FROM employees +SELECT name, SUM(salary) OVER w, RANK() OVER (w ORDER BY salary) + FROM employees WINDOW w AS (PARTITION BY department) ``` -Supported functions: `ROW_NUMBER`, `RANK`, `DENSE_RANK`, `LAG`, `LEAD`, and the -aggregates `COUNT`, `SUM`, `AVG`, `MIN`, `MAX`. - -The frame follows the standard: **without** `ORDER BY` in the `OVER` clause the -frame is the whole partition, so every row sees the partition total; **with** -`ORDER BY` the frame grows row by row, giving a running value. +Supported functions: + +| Function | Value | +|---|---| +| `ROW_NUMBER()` | 1, 2, ... within the partition | +| `RANK()` | 1 + number of rows before the current row's peers (gaps after ties) | +| `DENSE_RANK()` | number of the current peer group (no gaps) | +| `PERCENT_RANK()` | (rank - 1) / (partition rows - 1); 0 for a one-row partition | +| `CUME_DIST()` | rows up to and including the last peer / partition rows | +| `NTILE(n)` | bucket 1..n; the first `rows % n` buckets get one row more | +| `LAG(x [, offset [, default]])` | `x` from `offset` (default 1) rows back, else `default` (NULL) | +| `LEAD(x [, offset [, default]])` | likewise, forward | +| `FIRST_VALUE(x)`, `LAST_VALUE(x)`, `NTH_VALUE(x, n)` | `x` at the first / last / n-th row of the frame | +| `COUNT(*)`, `COUNT(x)`, `SUM`, `AVG`, `MIN`, `MAX` | the aggregate over the frame | + +Partition and order keys may be any expressions, any number of them; NULLs +sort first. Rows with equal ORDER BY keys are *peers*. A query may use any +number of window functions with different `OVER` clauses, and window calls +may appear inside expressions (`ROUND(AVG(x) OVER (...), 1)`) and in ORDER BY. +Windows work over joins, over grouped rows (they run after GROUP BY and +HAVING), and in subqueries. A window function in WHERE, GROUP BY, HAVING or +inside an aggregate is an error (*misuse of window function*). + +The frame (the rows an aggregate or FIRST_VALUE / LAST_VALUE / NTH_VALUE +sees) defaults, as in SQLite, to `RANGE BETWEEN UNBOUNDED PRECEDING AND +CURRENT ROW`: **without** `ORDER BY` that is the whole partition, so every row +sees the partition total; **with** `ORDER BY` it is a running value in which +peers share the same value. ```sql -- 210000 on every Engineering row @@ -206,7 +372,14 @@ SELECT name, SUM(salary) OVER (PARTITION BY department) FROM employees SELECT name, SUM(salary) OVER (PARTITION BY department ORDER BY salary) FROM employees ``` -Explicit frame clauses (`ROWS BETWEEN ...`) are not supported. +Explicit frames: `ROWS` with any of `UNBOUNDED PRECEDING`, `n PRECEDING`, +`CURRENT ROW`, `n FOLLOWING`, `UNBOUNDED FOLLOWING` (`n` a non-negative +integer literal), and `RANGE` with `UNBOUNDED` / `CURRENT ROW` bounds. Named +windows (`WINDOW w AS (...)`, `OVER w`, `OVER (w ORDER BY ...)`) are +supported. Not supported, and rejected with an error: `RANGE` frames with an +offset, `GROUPS` frames, `EXCLUDE`, `FILTER` on a window function, and +`DISTINCT` inside one. SUM of integers is an integer; AVG, PERCENT_RANK and +CUME_DIST are floats. ## Set Operations @@ -225,6 +398,9 @@ INSERT INTO table (col1, col2) VALUES (v1, v2) INSERT INTO table SELECT ... FROM other_table ``` +The SELECT may be any query (joins, GROUP BY, ORDER BY/LIMIT, UNION, ...). +It must yield one column per target column. + ## UPDATE ```sql @@ -232,17 +408,17 @@ UPDATE table SET col1 = val1 [, col2 = val2, ...] [WHERE condition] ``` -`UPDATE ... FROM other_table` is not supported and is accepted without effect. +`UPDATE ... FROM other_table` is not supported and is rejected with an error. To pull a value from another table, use a correlated subquery in the SET -expression -- it must select an aggregate: +expression: ```sql UPDATE data SET category = - (SELECT MAX(category) FROM lookup WHERE lookup.code = data.code) + (SELECT category FROM lookup WHERE lookup.code = data.code) ``` -A correlated scalar subquery selecting a bare column is rejected with -*"Correlated scalar subquery must select an aggregate function"*. +When several lookup rows match, the first is used; a row with no match gets +NULL. ## DELETE @@ -267,6 +443,10 @@ CREATE TABLE name ( CREATE TABLE name AS SELECT ... FROM ... ``` +The new table's columns are the query's result columns. Column and table +constraints (`PRIMARY KEY`, `NOT NULL`, `DEFAULT`, ...) are rejected: sqawk +does not enforce them. + ## DROP TABLE ```sql @@ -288,7 +468,8 @@ TRUNCATE TABLE name ## NULL handling -An empty field reads as NULL. +An empty field reads as NULL, and NULL is printed (and written back) as an +empty field. Comparison follows SQL three-valued logic: any comparison involving NULL is UNKNOWN, and a row whose `WHERE` evaluates to UNKNOWN is not returned. This @@ -296,7 +477,11 @@ means a NULL row satisfies neither `x > 5` nor `x <= 5`, and `NULL = NULL` is UNKNOWN rather than true. Use `IS NULL` / `IS NOT NULL` to test for NULL. Aggregates skip NULLs: `COUNT(*)` counts rows while `COUNT(col)` counts -non-NULL values. `ORDER BY` sorts NULLs first. +non-NULL values. `ORDER BY` sorts NULLs first ascending and last descending, +unless the term says `NULLS FIRST` or `NULLS LAST` (also in a window's ORDER +BY). + +NULL and the empty string both print as an empty field. ## Type coercion @@ -306,8 +491,68 @@ floats and text. When a comparison mixes a number and a string, the string is converted to a number if it parses as one, and the two are compared textually otherwise. This matches arithmetic, so `x + '1'` and `x > '1'` agree — awk-like rather than -strict SQL, which suits untyped delimited input. +strict SQL, which suits untyped delimited input. Two strings compare as text. + +Text that spells a number (`007`, `1.50`, `1e3`, which load as text to keep +their spelling) is used as that number by arithmetic and by `SUM` and `AVG`. +Arithmetic reads any text by its numeric prefix (see Arithmetic). In a +condition (WHERE, `NOT`, `AND`, CASE), text is likewise read as a number, so +`'x'` is false and `'1x'` true, as in SQLite. +`ORDER BY`, `GROUP BY`, `MIN` and `MAX` order such text among the numbers by +value; values equal as numbers but spelled differently (`7` and `007`) remain +distinct groups. ## Writeback Modifications (INSERT, UPDATE, DELETE) remain in-memory unless `--write` flag is specified. +Cells a statement did not change are written back byte for byte; NULL is +written as an empty field; the source file's delimiter, header line (or its +absence), line endings, `#` comment lines (in a non-`.csv` file) and +permissions are kept. + +## Unsupported syntax + +The SQL parser accepts many dialects. Anything it accepts that sqawk does not +implement is an error ("... is not supported"), never silently ignored, and +nothing is modified. That includes: + +- `INSERT OR REPLACE` / `OR IGNORE` / ..., `ON CONFLICT`, `INSERT OVERWRITE`, + and `RETURNING` on INSERT, UPDATE and DELETE (`REPLACE INTO` does not parse) +- `UPDATE ... FROM`, `DELETE ... USING`, `DELETE t FROM t JOIN ...`, and + UPDATE / DELETE with `ORDER BY` or `LIMIT` +- `WITH` (common table expressions), `FETCH FIRST`, `FOR UPDATE`, `TOP`, + `QUALIFY`, `SORT BY` / `CLUSTER BY` / `DISTRIBUTE BY`, `LATERAL VIEW`, + `SELECT ... INTO`, `GROUP BY ... WITH ROLLUP` +- `TABLESAMPLE`, table hints, `LATERAL` subqueries, column aliases on a FROM + item +- `WITHIN GROUP`, `IGNORE NULLS`, `ORDER BY` inside an aggregate's argument + list, `FILTER` or `DISTINCT` on a scalar function, `FILTER` on a window + function +- `CAST ... FORMAT`, `LIKE ANY`, blob literals (`X'00'`) and parameters +- table and column constraints, `CREATE TABLE IF NOT EXISTS`, + `PARTITIONED BY`, `ROW FORMAT` +- statements other than SELECT, INSERT, UPDATE, DELETE, CREATE TABLE, DROP + TABLE, ALTER TABLE ADD COLUMN and TRUNCATE (no views, indexes, triggers or + transactions) + +## Differences from SQLite + +sqawk aims to give SQLite's answers. Where it deliberately does not: + +| Area | SQLite | sqawk | +|------|--------|-------| +| Column types | Declared affinity converts `'10'` stored in an INTEGER column to 10 | Each cell is typed when read; a text/number comparison converts the text if it is a number (`a = '10'` matches 10) -- awk-like, see Type coercion | +| Real output | `%!.15g`: `1.0`, `9.22337203685478e+18` | Shortest round-trip digits, no exponent, no `.0`: `1`, `9223372036854775808` | +| NULL vs `''` | Distinct in output | Both print as an empty field | +| `UPPER` / `LOWER` | ASCII letters only | All of Unicode | +| `TRIM(s)` | Strips spaces only | Strips all whitespace | +| `ROUND(x, -d)` | Negative `d` treated as 0 | Rounds to the left of the point | +| `SUM` over text | Non-numeric text counts as 0 (result real) | Non-numeric text is skipped | +| `INSERT OR REPLACE`, `ON CONFLICT` | Plain INSERT when the table has no unique constraint | Rejected: sqawk tables have no keys, so the statement cannot do what it asks | +| Subqueries in UPDATE / DELETE | A correlated subquery sees rows already changed | Sees the table as it was before the statement | +| `ILIKE`, `LPAD`, `RPAD`, `LEFT`, `RIGHT`, `TRIM(... FROM ...)` | Not available | Available | +| `localtime` date modifier | Local time zone | Error (UTC only) | +| Parser | SQLite grammar | sqlparser's Hive dialect: `GLOB`, `REGEXP`, `x IS 10`, `LIMIT n, m`, `TRIM(s, chars)`, `REPLACE INTO` do not parse | + +Not implemented: common table expressions, views, indexes, triggers, +`RETURNING`, row values, `GLOB`, and blob values. diff --git a/doc/user_guide.md b/doc/user_guide.md index 183a597..c68b229 100644 --- a/doc/user_guide.md +++ b/doc/user_guide.md @@ -93,10 +93,11 @@ To build from source: - Sqawk is designed to work on Linux, macOS, and Windows systems where Rust is supported - Building requires Rust 1.88 or newer - The binaries are self-contained with no runtime dependencies -- Installing the crate provides **two** commands: `sqawk`, and `tsq`, a - generator of deterministic multi-table CSV data and SQL queries for - exercising sqawk. Run `tsq --help`, or see the - [project README](../README.md#tsq---test-data-generator) +- Installing the crate provides the `sqawk` command. The source tree also + contains `tsq`, a development-only generator of deterministic multi-table + CSV data and SQL queries for exercising sqawk; it is not installed. Run + `cargo run -p tsq -- --help` from a checkout, or see the + [project README](../README.md#tsq---test-data-generator-development-only) ## Getting Started @@ -145,8 +146,12 @@ When you run a Sqawk command: 1. Sqawk loads each specified file into memory as a table 2. Table names are derived from file names (without extensions) or can be explicitly assigned -3. The first row is treated as column headers -4. Data types are automatically inferred (numbers, strings, etc.) +3. The first row is treated as column headers (always for `.csv` files and + standard input; other files are checked for a missing header -- see + [Header Rows](#header-rows)) +4. Data types are inferred per cell: an empty field is NULL, a number written + in its canonical form (`42`, `1.5`) is a number, and everything else is + text, kept exactly as written (`007`, `1.50`, `1e3`, `true`) 5. SQL queries are executed against the in-memory tables 6. Results are displayed on the console 7. If `--write` is specified, modified tables are saved back to the source files @@ -266,8 +271,14 @@ sqawk -s "DELETE FROM data WHERE status = 'expired'" data.csv --write Important notes about the write behavior: - Only tables that were actually modified by an operation (INSERT, UPDATE, DELETE) are saved -- The original file format and delimiter are preserved -- Column order and headers are maintained +- Cells the statement did not change are written back exactly as they were + read: `007` stays `007`, `1.50` stays `1.50`, quoted text keeps its quotes + where they are needed. The one normalisation is that quotes a field did + not need (`"plain"`) are dropped +- The delimiter, the header line (or its absence), CRLF line endings, the + file's permissions and row order are preserved. Writing through a symlink + updates the file it points to and leaves the link in place +- NULL is written as an empty field - Without `--write`, your files remain untouched regardless of the SQL operations ### Field Separator Option (-F) @@ -287,6 +298,9 @@ Notes on field separators: tab**, `.tsv` and `.txt` alike. A comma-separated `.txt` file therefore needs an explicit `-F,`. A file with no extension at all defaults to comma - Common separators include tab (`\t`), comma (`,`), colon (`:`), and pipe (`|`) +- The separator must be a single ASCII character other than a quote or line + break; the two-character escape `\t` means tab. Anything else (including an + empty `-F ''`) is rejected with an error - `-F` applies to every input file in the invocation, not per file - The input separator is preserved when writing a table back with `--write` - Query results on stdout use `-F` when it is given, and comma otherwise -- @@ -320,6 +334,24 @@ sqawk -F: \ passwd=/etc/passwd group=/etc/group ``` +### Header Rows + +- A `.csv` file's first line is always its header, as is the first line of + standard input. Lines starting with `#` are data +- Any other file (`.tsv`, `.txt`, no extension) is checked: if its first line + looks like data rather than column names -- a number, a path starting with + `/`, `*`, `root` or `nobody` -- it is treated as data and the columns are + named `a`, `b`, `c`, ... Lines starting with `#` are comments and skipped +- `--no-header` says no input file has a header row: the first line is data + and the columns are named `a`, `b`, `c`, ... (or by `--tabledef`) +- `--tabledef` likewise means the file has no header row + +A table loaded without a header row is written back without one. + +```sh +sqawk --no-header -s "SELECT a, b FROM pairs WHERE b > 10" pairs.csv +``` + ### Verbose Mode (-v) The verbose mode provides additional information about the operations being performed: @@ -328,7 +360,8 @@ The verbose mode provides additional information about the operations being perf sqawk -s "SELECT * FROM data WHERE id > 1000" data.csv -v ``` -Verbose output includes: +Verbose output goes to standard error, so it never mixes with query results. +It includes: - SQL statements being executed - Number of rows affected or returned - Table loading information @@ -362,9 +395,11 @@ File format detection follows these rules: #### Comment Support -Sqawk supports comment lines in CSV and other delimiter-separated files. Lines that begin with a comment character (typically '#') are ignored during processing: +Delimited files other than `.csv` may contain comment lines. Lines that begin +with `#` are ignored when such a file is read. In a `.csv` file, a `#` line is +an ordinary record: -```csv +``` # This line is a comment and will be ignored id,name,department,salary 1,Alice,Engineering,75000 @@ -379,21 +414,22 @@ Comment support is useful for: #### Rows With The Wrong Field Count -A row whose field count does not match the header is reconciled to the header -rather than rejected. There is no flag for this and no strict mode: - - A row with **too few** fields is padded with NULL -- A row with **too many** fields is truncated to the header width +- A row with **too many** fields is an error naming the line, since there is + no column to hold the extra values and dropping them would lose data ```csv a,b,c 1,2 # loads as 1, 2, NULL -1,2,3,4 # loads as 1, 2, 3 +1,2,3,4 # error: record has 4 fields, but the table has 3 columns ``` -This keeps imperfect data sources usable, but it is silent: nothing warns that -a row was padded or truncated. If a query returns unexpected NULLs in the last -columns, check the input for short rows. +Padding is silent. If a query returns unexpected NULLs in the last columns, +check the input for short rows. + +Fields are read per RFC 4180: a quoted field may contain the delimiter, line +breaks, and doubled quotes (`"say ""hi"""`). Fields are never trimmed, and a +UTF-8 byte order mark at the start of the file is ignored. Completely empty input is a different matter. The first line is the header, so a file or pipe with no content has no column names and is rejected with *"has @@ -546,10 +582,11 @@ Sqawk follows a safe-by-default approach to file modification: - Files are never modified unless the `--write` flag is provided - Only tables that were actually changed are written back - When writing back: - - Original delimiters and formatting are preserved - - Column order remains the same - - Header row is preserved - - Empty values are written as empty fields, not NULLs + - Unchanged cells are written back byte for byte + - Original delimiter, header line (or its absence) and line endings are preserved + - Column and row order remain the same + - NULL is written as an empty field + - The file keeps its permissions; a symlink keeps pointing at the rewritten file Example of safe write behavior: @@ -562,19 +599,16 @@ sqawk -s "DELETE FROM data WHERE code NOT IN (SELECT code FROM lookup)" \ `UPDATE ... FROM other_table` is **not** supported, and is silently accepted without doing anything -- no error, no rows changed. A cross-table update has -to go through a correlated subquery in the SET expression, which must select an -aggregate: +to go through a correlated subquery in the SET expression: ```sh sqawk -s "UPDATE data SET category = - (SELECT MAX(category) FROM lookup WHERE lookup.code = data.code)" \ + (SELECT category FROM lookup WHERE lookup.code = data.code)" \ data.csv lookup.csv --write ``` -The `MAX()` is not cosmetic: a correlated scalar subquery that selects a bare -column is rejected with *"Correlated scalar subquery must select an aggregate -function"*. Wrapping the column in `MAX` picks the single matching value when -the lookup key is unique. +A row with no matching lookup row gets NULL (written back as an empty field); +add `WHERE code IN (SELECT code FROM lookup)` to leave such rows untouched. ## Examples @@ -664,8 +698,24 @@ sqawk -s "SELECT SUBSTR(date, 1, 7) AS month FROM transactions" transactions.csv # String replacement sqawk -s "UPDATE data SET phone = REPLACE(phone, '-', '')" data.csv --write + +# List each department's members on one line +sqawk -s "SELECT dept, GROUP_CONCAT(name, '; ') FROM staff GROUP BY dept" staff.csv ``` +### SQL Semantics + +Sqawk answers as SQLite does, with a few documented exceptions (see "Differences +from SQLite" in the [SQL reference](sql_reference.md)): + +- Integer arithmetic stays integer: `5 / 2` is `2`, `7 % 3` is `1`; overflow + becomes a real, and division by zero is NULL. +- `LIKE` is case-insensitive for ASCII letters; `ILIKE` folds all of Unicode. +- String functions count characters, not bytes; `'a' || NULL` is NULL. +- Dates are UTC: `DATE('now')`, `DATE(d, '+1 month')`, `STRFTIME('%Y', d)`. +- Syntax that parses but is not implemented (`RETURNING`, `WITH`, + `UPDATE ... FROM`, ...) is an error, never silently ignored. + ### System Files (--tabledef) ```sh @@ -698,6 +748,13 @@ The output delimiter is whatever `-F` says, and comma when `-F` is absent. It does not follow the input file's delimiter, which is what makes the conversion above work. +Output is a well-formed delimited file: a field containing the delimiter, a +quote or a line break, or with leading or trailing spaces, is quoted +(RFC 4180, with embedded quotes doubled), and NULL is an empty field. A row +consisting of a single NULL prints as `""` so that it is not a blank line. +When the reader goes away (`sqawk ... | head`), sqawk stops printing quietly +and exits successfully. + ### Reading Standard Input A file operand of `-` reads standard input, so sqawk can sit in the middle of a @@ -769,10 +826,8 @@ An unaliased aggregate takes the function's name as its column header -- ## Working with Large Files -Sqawk holds a whole table at once. On-disk files are memory-mapped, so loading -is zero-copy and read-only queries allocate almost nothing; a table is copied to -the heap the first time it is modified. Either way the dataset must fit, which -is worth planning for with large files: +Sqawk holds a whole table in memory at once, so the dataset must fit, which is +worth planning for with large files: **Tips for handling large files:** diff --git a/src/aggregate.rs b/src/aggregate.rs index 84a9f99..c6e8f3a 100644 --- a/src/aggregate.rs +++ b/src/aggregate.rs @@ -1,18 +1,8 @@ -//! Aggregate function module for sqawk +//! Aggregate function names. //! -//! This module implements SQL aggregate functions (COUNT, SUM, AVG, MIN, MAX) for the sqawk utility. -//! It provides the core aggregation functionality for SQL queries with features including: -//! -//! - Type-aware aggregation that preserves the appropriate result type (integer vs. float) -//! - Proper NULL handling according to SQL semantics -//! - Support for mixed-type aggregation with type coercion -//! - Comprehensive test coverage for all aggregate operations -//! -//! All aggregate functions operate on columns (vectors of values) and return a single -//! Value representing the result of the operation. - -use crate::error::SqawkResult; -use crate::table::Value; +//! Aggregates are compiled by `vm::compiler_aggregate` and evaluated by the +//! VM's `AggStep` / `AggFinal` opcodes; this module only recognizes the names +//! and maps them to their VM function type. /// Supported aggregate functions #[derive(Debug, Clone, Copy)] @@ -27,6 +17,10 @@ pub enum AggregateFunction { Min, /// MAX function - finds the maximum value in a column Max, + /// TOTAL function - SUM as a real, 0.0 over no rows + Total, + /// GROUP_CONCAT / STRING_AGG - values joined by a separator + GroupConcat, } impl AggregateFunction { @@ -44,269 +38,52 @@ impl AggregateFunction { "AVG" => Some(AggregateFunction::Avg), "MIN" => Some(AggregateFunction::Min), "MAX" => Some(AggregateFunction::Max), + "TOTAL" => Some(AggregateFunction::Total), + "GROUP_CONCAT" | "STRING_AGG" => Some(AggregateFunction::GroupConcat), _ => None, } } - /// Execute the aggregate function on a column of values - /// - /// # Arguments - /// * `values` - The column values to aggregate - /// - /// # Returns - /// * The result of the aggregate function - pub fn execute(&self, values: &[Value]) -> SqawkResult { + /// The AggStep/AggFinal function-type constant for this aggregate. + pub fn vm_type(self) -> i64 { + use crate::vm::bytecode::{ + AGG_AVG, AGG_COUNT, AGG_GROUP_CONCAT, AGG_MAX, AGG_MIN, AGG_SUM, AGG_TOTAL, + }; match self { - AggregateFunction::Count => self.count(values), - AggregateFunction::Sum => self.sum(values), - AggregateFunction::Avg => self.avg(values), - AggregateFunction::Min => self.min(values), - AggregateFunction::Max => self.max(values), + AggregateFunction::Count => AGG_COUNT, + AggregateFunction::Sum => AGG_SUM, + AggregateFunction::Avg => AGG_AVG, + AggregateFunction::Min => AGG_MIN, + AggregateFunction::Max => AGG_MAX, + AggregateFunction::Total => AGG_TOTAL, + AggregateFunction::GroupConcat => AGG_GROUP_CONCAT, } } - /// COUNT function implementation - /// - /// Counts non-NULL values in the column - fn count(&self, values: &[Value]) -> SqawkResult { - let count = values.iter().filter(|v| !matches!(v, Value::Null)).count(); - - Ok(Value::Integer(count as i64)) - } - - /// SUM function implementation - /// - /// Sums numeric values in the column - fn sum(&self, values: &[Value]) -> SqawkResult { - // Keep track of whether we need to return an integer or float - let mut is_float = false; - let mut int_sum: i64 = 0; - let mut float_sum: f64 = 0.0; - - // Count non-null values - let mut count = 0; - - for value in values { - match value { - Value::Integer(i) => { - if is_float { - float_sum += *i as f64; - } else { - int_sum += *i; - } - count += 1; - } - Value::Float(f) => { - if !is_float { - // Convert accumulated integer sum to float - float_sum = int_sum as f64; - is_float = true; - } - float_sum += *f; - count += 1; - } - // Ignore non-numeric values - _ => {} - } - } - - // If no numeric values were found, return NULL - if count == 0 { - return Ok(Value::Null); - } - - // Return the sum in the appropriate type - if is_float { - Ok(Value::Float(float_sum)) - } else { - Ok(Value::Integer(int_sum)) - } - } - - /// AVG function implementation - /// - /// Calculates the average of numeric values in the column - fn avg(&self, values: &[Value]) -> SqawkResult { - // Get the sum first - let sum = self.sum(values)?; - - // Count numeric values - let count = values - .iter() - .filter(|v| matches!(v, Value::Integer(_) | Value::Float(_))) - .count(); - - if count == 0 { - return Ok(Value::Null); - } - - // Convert to float and divide by count - match sum { - Value::Integer(i) => Ok(Value::Float(i as f64 / count as f64)), - Value::Float(f) => Ok(Value::Float(f / count as f64)), - _ => Ok(Value::Null), // This shouldn't happen, but handle it just in case - } - } - - /// MIN function implementation - /// - /// Finds the minimum value in the column - fn min(&self, values: &[Value]) -> SqawkResult { - // Filter out NULL values - let non_null_values: Vec<&Value> = values - .iter() - .filter(|v| !matches!(v, Value::Null)) - .collect(); - - if non_null_values.is_empty() { - return Ok(Value::Null); - } - - // Start with the first value as our minimum - let mut min_value = non_null_values[0].clone(); - - // Compare with the rest of the values - for value in &non_null_values[1..] { - if value < &&min_value { - min_value = (*value).clone(); - } + /// The argument counts the aggregate accepts (COUNT(*) counts as one). + pub fn arity(self, name: &str) -> (usize, usize) { + match self { + AggregateFunction::GroupConcat if name.eq_ignore_ascii_case("STRING_AGG") => (2, 2), + AggregateFunction::GroupConcat => (1, 2), + _ => (1, 1), } - - Ok(min_value) } - /// MAX function implementation + /// Whether `name` (any case) is an aggregate function. /// - /// Finds the maximum value in the column - fn max(&self, values: &[Value]) -> SqawkResult { - // Filter out NULL values - let non_null_values: Vec<&Value> = values - .iter() - .filter(|v| !matches!(v, Value::Null)) - .collect(); - - if non_null_values.is_empty() { - return Ok(Value::Null); - } - - // Start with the first value as our maximum - let mut max_value = non_null_values[0].clone(); - - // Compare with the rest of the values - for value in &non_null_values[1..] { - if value > &&max_value { - max_value = (*value).clone(); - } - } - - Ok(max_value) - } -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn test_count_function() { - let values = vec![ - Value::Integer(10), - Value::Null, - Value::Integer(20), - Value::String("test".to_string().into()), - ]; - - let count = AggregateFunction::Count.execute(&values).unwrap(); - assert_eq!(count, Value::Integer(3)); - } - - #[test] - fn test_sum_function() { - let values = vec![ - Value::Integer(10), - Value::Null, - Value::Integer(20), - Value::Float(5.5), - ]; - - let sum = AggregateFunction::Sum.execute(&values).unwrap(); - assert_eq!(sum, Value::Float(35.5)); + /// The single source of the aggregate name list: every site that needs to + /// recognise an aggregate asks here. + pub fn is_aggregate_name(name: &str) -> bool { + Self::from_name(name).is_some() } - #[test] - fn test_sum_integers_only() { - let values = vec![Value::Integer(10), Value::Integer(20), Value::Integer(30)]; - - let sum = AggregateFunction::Sum.execute(&values).unwrap(); - assert_eq!(sum, Value::Integer(60)); - } - - #[test] - fn test_avg_function() { - let values = vec![ - Value::Integer(10), - Value::Null, - Value::Integer(20), - Value::Float(30.0), - ]; - - let avg = AggregateFunction::Avg.execute(&values).unwrap(); - if let Value::Float(f) = avg { - assert!((f - 20.0).abs() < f64::EPSILON); - } else { - panic!("Expected Float, got {:?}", avg); + /// Whether a call of `name` with `nargs` arguments is an aggregate. + /// MIN and MAX with two or more arguments are the scalar functions. + pub fn is_aggregate_call(name: &str, nargs: usize) -> bool { + match Self::from_name(name) { + Some(AggregateFunction::Min | AggregateFunction::Max) => nargs < 2, + Some(_) => true, + None => false, } } - - #[test] - fn test_min_function() { - let values = vec![ - Value::Integer(30), - Value::Integer(10), - Value::Null, - Value::Integer(20), - ]; - - let min = AggregateFunction::Min.execute(&values).unwrap(); - assert_eq!(min, Value::Integer(10)); - } - - #[test] - fn test_min_with_mixed_types() { - let values = vec![ - Value::Integer(30), - Value::Float(5.5), - Value::String("abc".to_string().into()), - Value::Null, - ]; - - let min = AggregateFunction::Min.execute(&values).unwrap(); - assert_eq!(min, Value::Float(5.5)); - } - - #[test] - fn test_max_function() { - let values = vec![ - Value::Integer(30), - Value::Integer(10), - Value::Null, - Value::Integer(20), - ]; - - let max = AggregateFunction::Max.execute(&values).unwrap(); - assert_eq!(max, Value::Integer(30)); - } - - #[test] - fn test_max_with_mixed_types() { - let values = vec![ - Value::Integer(30), - Value::Float(50.5), - Value::String("xyz".to_string().into()), - Value::Null, - ]; - - let max = AggregateFunction::Max.execute(&values).unwrap(); - assert_eq!(max, Value::String("xyz".to_string().into())); - } } diff --git a/src/capacity.rs b/src/capacity.rs index aa490ed..0389876 100644 --- a/src/capacity.rs +++ b/src/capacity.rs @@ -7,10 +7,6 @@ // Row and Column Capacities // ============================================================================= -/// Default capacity for row vectors (number of fields per row). -/// Most CSV/TSV files have fewer than 64 columns. -pub const DEFAULT_ROW_CAPACITY: usize = 16; - /// Default capacity for column definition vectors. pub const DEFAULT_COLUMN_CAPACITY: usize = 16; @@ -67,39 +63,10 @@ pub const DEFAULT_SORTER_ROWS_CAPACITY: usize = 256; /// Default capacity for sort key vectors. pub const DEFAULT_SORT_KEYS_CAPACITY: usize = 4; -// ============================================================================= -// File-based Estimation -// ============================================================================= - -/// Average bytes per row estimate for CSV files. -/// Used to estimate row count from file size. -pub const ESTIMATED_BYTES_PER_ROW: usize = 128; - -/// Minimum row capacity for file-based estimation. -pub const MIN_ESTIMATED_ROW_CAPACITY: usize = 64; - -/// Maximum row capacity for file-based estimation. -/// Prevents excessive pre-allocation for very large files. -pub const MAX_ESTIMATED_ROW_CAPACITY: usize = 10_000_000; - -/// Estimate row count from file size (for mmap pre-allocation). -/// Returns a capacity hint, not an exact count. -#[inline] -pub fn estimate_row_count(file_size: usize) -> usize { - let estimate = file_size / ESTIMATED_BYTES_PER_ROW; - estimate.clamp(MIN_ESTIMATED_ROW_CAPACITY, MAX_ESTIMATED_ROW_CAPACITY) -} - // ============================================================================= // Headerless File Detection // ============================================================================= -/// Check if a line is a comment (starts with #). -#[inline] -pub fn is_comment_line(line: &[u8]) -> bool { - line.starts_with(b"#") -} - /// Check if a field looks like data rather than a header name. /// Used to auto-detect headerless files like /etc/passwd. #[inline] diff --git a/src/cli.rs b/src/cli.rs index 4b100cd..cf345a2 100644 --- a/src/cli.rs +++ b/src/cli.rs @@ -94,6 +94,13 @@ pub struct SqawkArgs { #[clap(short = 'F', help = "Field separator character")] pub field_separator: Option, + /// Input files have no header row + /// + /// The first line of every input file is data. Columns are named a, b, + /// c, ... unless --tabledef names them. Writeback emits no header line. + #[clap(long = "no-header", help = "Treat the first line of each file as data")] + pub no_header: bool, + /// Enable verbose diagnostic output /// /// When enabled, shows detailed information about SQL execution, @@ -123,3 +130,22 @@ pub struct SqawkArgs { pub fn parse_args() -> Result { Ok(SqawkArgs::parse()) } + +/// Validate a `-F` field separator and return it in canonical form. +/// +/// The separator must be exactly one ASCII byte, other than a quote or a +/// line break; the two-character escape `\t` is accepted as a tab. +pub fn validate_separator(sep: &str) -> Result { + let sep = if sep == "\\t" { "\t" } else { sep }; + match sep.as_bytes() { + [] => anyhow::bail!("field separator (-F) must not be empty"), + [b'"' | b'\n' | b'\r'] => { + anyhow::bail!("field separator (-F) cannot be a quote or a line break") + } + [b] if b.is_ascii() => Ok(sep.to_string()), + _ => anyhow::bail!( + "field separator (-F) must be a single ASCII character (or \\t for tab), got {:?}", + sep + ), + } +} diff --git a/src/config.rs b/src/config.rs index dca9aa0..1a342fa 100644 --- a/src/config.rs +++ b/src/config.rs @@ -23,6 +23,9 @@ pub struct AppConfig { /// Whether to write changes back to files write_changes: bool, + + /// Treat the first line of every input file as data (`--no-header`) + no_header: bool, } impl AppConfig { @@ -44,9 +47,21 @@ impl AppConfig { field_separator, table_definitions, write_changes, + no_header: false, } } + /// Set whether input files have no header row + pub fn with_no_header(mut self, no_header: bool) -> Self { + self.no_header = no_header; + self + } + + /// Whether input files have no header row + pub fn no_header(&self) -> bool { + self.no_header + } + /// Get the verbose flag pub fn verbose(&self) -> bool { self.verbose diff --git a/src/csv_handler.rs b/src/csv_handler.rs index 8bf8cf0..2405e76 100644 --- a/src/csv_handler.rs +++ b/src/csv_handler.rs @@ -1,31 +1,352 @@ -//! CSV file handling module for sqawk +//! Delimited file reading and writing for sqawk //! -//! This module handles loading CSV files into in-memory tables and saving tables back to CSV files. -//! It provides functionality for: +//! Every input -- `.csv`, TSV, `-F`-delimited files and standard input -- is +//! parsed here by the `csv` crate with one configuration: RFC 4180 quoting +//! (`""` inside a quoted field is a literal quote), quoted fields may span +//! lines, fields are never trimmed, a UTF-8 byte order mark is stripped, and +//! rows may be ragged (short rows are padded with NULL). //! -//! - Loading CSV files with automatic header detection -//! - Parsing file specifications in the format [table_name=]file_path.csv -//! - Managing a collection of in-memory tables -//! - Converting between CSV records and the internal Value type -//! - Writing modified tables back to CSV files -//! -//! The module uses buffered I/O operations for efficiency and maintains -//! a mapping between table names and their source files for writeback operations. - -use std::fs::File; -use std::io::BufReader; +//! Writeback goes through the same module, so that a file read and written +//! back without changes is byte-identical: cells are inferred with a rule +//! that round-trips (see `Value::from(&str)`), NULL is the empty field, and +//! the header/line-ending/comment layout of the source is recorded on the +//! table and reproduced, including the `#` comment lines themselves (see +//! `CommentLines`). + +use std::io::{BufWriter, Write}; use std::path::{Path, PathBuf}; use tempfile::NamedTempFile; +use crate::capacity::{generate_alpha_columns, is_likely_data_row}; use crate::error::{SqawkError, SqawkResult}; -use crate::table::{Table, Value}; +use crate::table::{CommentLines, FileLayout, Table, Value}; + +/// How the first record of a file is treated. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum HeaderMode { + /// The first record is the header (the rule for `.csv` and stdin). + Present, + /// Every record is data; columns are named a, b, c, ... unless given. + Absent, + /// Guess from the first record (`is_likely_data_row`); used for + /// non-.csv delimited files such as /etc/passwd. + Guess, +} + +/// Options for reading one delimited file. +#[derive(Debug, Clone)] +pub struct ReadOptions { + /// Field separator byte + pub delimiter: u8, + /// Header policy + pub header: HeaderMode, + /// Skip lines that start with `#` + pub comments: bool, + /// Column names to use; implies the file has no header row + pub columns: Option>, + /// Skip or repair malformed records instead of failing + pub recover_errors: bool, +} + +impl Default for ReadOptions { + /// Plain CSV: comma, header row, no comment lines. + fn default() -> Self { + ReadOptions { + delimiter: b',', + header: HeaderMode::Present, + comments: false, + columns: None, + recover_errors: false, + } + } +} + +/// UTF-8 byte order mark +const BOM: &[u8] = b"\xef\xbb\xbf"; + +/// Field bytes to text. Invalid UTF-8 is replaced rather than rejected. +fn field_text(bytes: &[u8]) -> std::borrow::Cow<'_, str> { + String::from_utf8_lossy(bytes) +} -/// Handles loading and saving CSV files +/// Parse delimited `data` into a table. /// -/// This struct provides methods for loading tables from CSV files -/// and writing them back when modified. It's specialized for handling -/// data in CSV format with commas as separators. +/// # Arguments +/// * `table_name` - name of the resulting table +/// * `data` - the whole file contents +/// * `file_path` - writeback target recorded on the table (None for stdin) +/// * `delimiter` - the delimiter string recorded on the table +/// * `opts` - parsing options +/// * `source` - file name used in error messages +pub fn read_table( + table_name: &str, + data: &[u8], + file_path: Option, + delimiter: String, + opts: &ReadOptions, + source: &str, +) -> SqawkResult { + let data = data.strip_prefix(BOM).unwrap_or(data); + + // Line ending of the first record decides the writeback terminator. + let crlf = data + .iter() + .position(|&b| b == b'\n') + .is_some_and(|i| i > 0 && data[i - 1] == b'\r'); + + let mut reader = csv::ReaderBuilder::new() + .has_headers(false) + .flexible(true) + .delimiter(opts.delimiter) + .comment(if opts.comments { Some(b'#') } else { None }) + .from_reader(data); + + let mut record = csv::ByteRecord::new(); + let mut line = 0usize; + let mut comment_lines = CommentLines::default(); + + // First record: the header, or the first data row. + let mut pending_first: Option> = None; + let first = read_record(&mut reader, &mut record, source, &mut line); + let mut has_first = first?; + if opts.comments { + let (lines, to_eof) = skipped_comments(data, &record); + comment_lines.head = lines; + has_first &= !(to_eof && is_phantom(&record)); + } + let header_mode = if opts.columns.is_some() { + HeaderMode::Absent + } else { + opts.header + }; + let (columns, has_header) = if !has_first { + (opts.columns.clone().unwrap_or_default(), false) + } else { + let fields: Vec = record.iter().map(|f| field_text(f).into_owned()).collect(); + let is_header = match header_mode { + HeaderMode::Present => true, + HeaderMode::Absent => false, + HeaderMode::Guess => !is_likely_data_row(&fields), + }; + if is_header { + (fields, true) + } else { + let cols = opts + .columns + .clone() + .unwrap_or_else(|| generate_alpha_columns(fields.len())); + pending_first = Some(fields.into_iter().map(Value::from).collect()); + (cols, false) + } + }; + + let mut table = Table::new_with_delimiter(table_name, columns, file_path, delimiter); + table.set_file_layout(FileLayout { + has_header, + crlf, + comments: opts.comments, + }); + let width = table.column_count(); + let mut sink = RowSink { + rows: Vec::new(), + width, + skipped: 0, + recover: opts.recover_errors, + source, + }; + if let Some(first) = pending_first { + sink.push(first, line)?; + } + + loop { + let mut next = read_record(&mut reader, &mut record, source, &mut line); + if opts.comments { + // The comment lines skipped before this record, or before the + // end of the file, go in front of the next row to be pushed (or + // into the tail, past the last row). + let (lines, to_eof) = skipped_comments(data, &record); + if !lines.is_empty() { + comment_lines + .before + .entry(sink.rows.len()) + .or_default() + .extend_from_slice(&lines); + } + if to_eof && matches!(next, Ok(true)) && is_phantom(&record) { + next = Ok(false); + } + } + match next { + Ok(true) => { + let row: Vec = record + .iter() + .map(|f| match std::str::from_utf8(f) { + Ok(s) => Value::from(s), + Err(_) => Value::from(field_text(f).into_owned()), + }) + .collect(); + sink.push(row, line)?; + } + Ok(false) => break, + Err(e) if opts.recover_errors => { + sink.skipped += 1; + eprintln!("Warning: Skipping malformed row: {}", e); + } + Err(e) => return Err(e), + } + } + + if sink.skipped > 0 { + eprintln!( + "Note: Skipped {} malformed rows while loading {}", + sink.skipped, source + ); + } + + // Loading is not a modification. + table.replace_rows(sink.rows); + table.set_comment_lines(comment_lines); + table.mark_unmodified(); + Ok(table) +} + +/// Collects loaded rows, enforcing the table width. +struct RowSink<'a> { + rows: Vec>, + width: usize, + skipped: usize, + recover: bool, + source: &'a str, +} + +impl RowSink<'_> { + fn push(&mut self, mut row: Vec, line: usize) -> SqawkResult<()> { + if row.len() > self.width { + if self.recover { + self.skipped += 1; + eprintln!( + "Warning: Skipping row at line {} of {}: {} fields, expected {}", + line, + self.source, + row.len(), + self.width + ); + return Ok(()); + } + return Err(SqawkError::CsvParseError { + file: self.source.to_string(), + line, + error: format!( + "record has {} fields, but the table has {} columns", + row.len(), + self.width + ), + }); + } + // A short record is padded: the missing trailing fields are NULL. + row.resize(self.width, Value::Null); + self.rows.push(row); + Ok(()) + } +} + +/// The `#` comment lines the reader skipped before `record` (which may be the +/// empty record of end of input), verbatim with their line endings. +/// +/// A record's position is where the reader stood when it started on it: +/// just past the previous record. What lies between there and the record +/// itself is blank lines (or the rest of the previous record's line ending) +/// and comment lines; the scan stops at the first line that is neither. +/// +/// Also returns whether the comment lines run to the end of the input. +fn skipped_comments(data: &[u8], record: &csv::ByteRecord) -> (Vec, bool) { + let Some(pos) = record.position() else { + return (Vec::new(), false); + }; + let mut i = (pos.byte() as usize).min(data.len()); + let mut out = Vec::new(); + while i < data.len() { + let end = data[i..] + .iter() + .position(|&b| b == b'\n') + .map_or(data.len(), |n| i + n + 1); + let line = &data[i..end]; + if line[0] == b'#' { + out.extend_from_slice(line); + } else if !line.iter().all(|&b| b == b'\r' || b == b'\n') { + break; + } + i = end; + } + (out, i >= data.len()) +} + +/// The csv crate reports a comment line that ends the input WITHOUT a line +/// ending as an empty record. It is not a row: the comment is all there is. +fn is_phantom(record: &csv::ByteRecord) -> bool { + record.len() == 1 && record[0].is_empty() +} + +/// Read the next record; false at end of input. `line` tracks the 1-based +/// line where the record starts, for error messages. +fn read_record( + reader: &mut csv::Reader<&[u8]>, + record: &mut csv::ByteRecord, + source: &str, + line: &mut usize, +) -> SqawkResult { + match reader.read_byte_record(record) { + Ok(more) => { + if let Some(pos) = record.position() { + *line = pos.line() as usize; + } + Ok(more) + } + Err(e) => Err(SqawkError::CsvParseError { + file: source.to_string(), + line: e.position().map_or(*line + 1, |p| p.line() as usize), + error: e.to_string(), + }), + } +} + +/// Write `table` back to `file_path`, reproducing the source layout. +/// +/// The write is atomic: a temp file in the target's directory is written, +/// synced, given the original file's permissions, and renamed over the +/// target. A symlink is resolved first, so the link survives and its target +/// is what gets rewritten. +pub fn save_table(table: &Table, file_path: &Path) -> SqawkResult<()> { + let target = match std::fs::canonicalize(file_path) { + Ok(real) => real, + // A file that does not exist yet (CREATE TABLE ... LOCATION). + Err(_) => file_path.to_path_buf(), + }; + let parent = match target.parent() { + Some(p) if !p.as_os_str().is_empty() => p, + _ => Path::new("."), + }; + let temp_file = NamedTempFile::new_in(parent)?; + + { + let mut out = BufWriter::new(temp_file.as_file()); + table.write_source_layout(&mut out)?; + out.flush()?; + } + + if let Ok(meta) = std::fs::metadata(&target) { + temp_file.as_file().set_permissions(meta.permissions())?; + } + temp_file.as_file().sync_all()?; + temp_file + .persist(&target) + .map_err(|e| SqawkError::IoError(e.error))?; + Ok(()) +} + +/// Loads and saves CSV files (library convenience API). +#[allow(dead_code)] // not used by the binary; part of the library API pub struct CsvHandler {} impl Default for CsvHandler { @@ -34,254 +355,231 @@ impl Default for CsvHandler { } } +#[allow(dead_code)] impl CsvHandler { /// Create a new CsvHandler - /// - /// # Returns - /// A new CsvHandler instance ready to load and manage CSV files pub fn new() -> Self { CsvHandler {} } - /// Save a table to a CSV file - /// - /// Uses atomic write pattern: writes to a temporary file first, then renames - /// into place. This preserves the original file if writing fails midway. - /// - /// # Arguments - /// * `table` - The table to save - /// * `file_path` - The path to the file - /// - /// # Returns - /// * `SqawkResult<()>` - Result of the operation + /// Save a table to a CSV file (see `save_table`) pub fn save_csv(&self, table: &Table, file_path: &Path) -> SqawkResult<()> { - // Create temp file in the same directory as target (required for atomic rename) - let parent = file_path.parent().unwrap_or(Path::new(".")); - let temp_file = NamedTempFile::new_in(parent).map_err(SqawkError::IoError)?; - - // Write data in a scope so the writer is dropped before we call persist - { - // Create a CSV writer writing to the temp file - let mut writer = csv::Writer::from_writer(temp_file.as_file()); - - // Write header row - writer - .write_record(table.columns()) - .map_err(SqawkError::CsvError)?; - - // Write data rows - for row in table.rows() { - let string_values: Vec = - row.iter().map(|value| value.to_string()).collect(); - - writer - .write_record(&string_values) - .map_err(SqawkError::CsvError)?; - } - - // Flush the CSV writer before it's dropped - writer.flush().map_err(SqawkError::IoError)?; - } - - // Sync to disk before rename to ensure data durability - temp_file - .as_file() - .sync_all() - .map_err(SqawkError::IoError)?; - - // Atomically rename temp file to target path - temp_file - .persist(file_path) - .map_err(|e| SqawkError::IoError(e.error))?; - - Ok(()) + save_table(table, file_path) } - /// Load a CSV file into an in-memory table + /// Load a CSV file into a table. /// - /// This method parses CSV files with header rows, creating tables with - /// appropriate column names and automatically inferring data types for each cell. + /// * `file_spec` - `[table_name=]file_path` + /// * `custom_columns` - column names to use; the file then has no header + /// row and its first line is data + /// * `recover_errors` - skip malformed records instead of failing /// - /// # Arguments - /// * `file_spec` - File specification in the format `[table_name=]file_path.csv` - /// If table_name is not specified, the file name without extension is used. - /// * `custom_columns` - Optional custom column names to use instead of detected/generated ones - /// * `recover_errors` - When true, malformed rows will be skipped instead of causing the operation to fail - /// - /// # Features - /// * Supports comments in CSV files (lines starting with #) - /// * Can recover from malformed rows by skipping them - /// * Provides detailed error information including line numbers - /// - /// # Returns - /// * `Ok(Table)` - The successfully loaded table - /// * `Err` if there was an error parsing the file spec, opening the file, or parsing the CSV data - /// - /// # Enhanced Error Handling - /// When the optional `recover_errors` parameter is set to true, the function will: - /// * Skip malformed rows instead of failing - /// * Log detailed error information including line numbers - /// * Continue processing the file to extract all valid rows + /// The first line is the header unless `custom_columns` is given. `#` + /// lines are data; use `load_with_options` for comment skipping. pub fn load_csv( &self, file_spec: &str, custom_columns: Option>, recover_errors: Option, ) -> SqawkResult
{ - // Parse file spec to get table name and file path - let (table_name, file_path) = self.parse_file_spec(file_spec)?; - - // Open the CSV file - let file = File::open(&file_path)?; - let reader = BufReader::new(file); - - // has_headers is true by default - first row is column names - let has_headers = true; - - // Create a CSV reader with enhanced options - let mut csv_reader = csv::ReaderBuilder::new() - .has_headers(has_headers) - .comment(Some(b'#')) // Support comment lines starting with # - // Enable flexible mode only if error recovery is requested - .flexible(true) // Always use flexible mode to allow for skipping errors - .from_reader(reader); - - // Get headers or use custom column names if provided - let headers = if let Some(columns) = custom_columns { - // Use the provided custom column names (first row will be data) - columns - } else { - // Use column names from the CSV header row - csv_reader - .headers() - .map_err(SqawkError::CsvError)? - .iter() - .map(|s| s.to_string()) - .collect::>() + let opts = ReadOptions { + columns: custom_columns, + recover_errors: recover_errors.unwrap_or(false), + ..ReadOptions::default() }; + self.load_with_options(file_spec, &opts) + } - // Create a new table with comma delimiter (since this is the CSV handler) - let mut table = Table::new_with_delimiter( + /// Load a delimited file with explicit options. + pub fn load_with_options(&self, file_spec: &str, opts: &ReadOptions) -> SqawkResult
{ + let (table_name, file_path) = parse_file_spec(file_spec)?; + let data = std::fs::read(&file_path)?; + let delimiter = (opts.delimiter as char).to_string(); + read_table( &table_name, - headers, + &data, Some(file_path.clone()), - ",".to_string(), + delimiter, + opts, + &file_path.to_string_lossy(), + ) + } +} + +/// Split `[table_name=]file_path`; the table name defaults to the file stem. +#[allow(dead_code)] +fn parse_file_spec(file_spec: &str) -> SqawkResult<(String, PathBuf)> { + if let Some((table_name, file_path)) = file_spec.split_once('=') { + return Ok((table_name.to_string(), PathBuf::from(file_path))); + } + let path = PathBuf::from(file_spec); + let stem = path + .file_stem() + .ok_or_else(|| { + SqawkError::InvalidFileSpec(format!("Invalid file specification: {}", file_spec)) + })? + .to_string_lossy() + .to_string(); + Ok((stem, path)) +} + +#[cfg(test)] +mod tests { + use super::{CsvHandler, HeaderMode, ReadOptions}; + use crate::error::SqawkError; + + /// Options for a comma-separated file with `#` comment lines. Plain `.csv` + /// loading treats `#` lines as data; comment skipping is opt-in (the CLI + /// enables it for non-.csv delimited files). + fn commented_options(columns: Option>) -> ReadOptions { + ReadOptions { + delimiter: b',', + header: HeaderMode::Present, + comments: true, + columns, + recover_errors: false, + } + } + + #[test] + fn test_csv_with_comments() { + // Create the CSV handler + let csv_handler = CsvHandler::new(); + + // Load the commented CSV file with comment skipping enabled + let table = csv_handler + .load_with_options("tests/data/commented.csv", &commented_options(None)) + .unwrap(); + + // Verify the correct data was loaded (comments should be ignored) + assert_eq!(table.rows().len(), 3); + assert_eq!(table.name(), "commented"); + + // Check specific row data + let rows = table.rows(); + assert_eq!(rows[0][1].to_string(), "Alice"); + assert_eq!(rows[1][1].to_string(), "Bob"); + assert_eq!(rows[2][1].to_string(), "Charlie"); + } + + #[test] + fn test_malformed_csv_with_error_recovery() { + // Create the CSV handler + let csv_handler = CsvHandler::new(); + + // Try to load the malformed CSV file with error recovery enabled + // We need to handle each row separately with flexible mode + let result = csv_handler.load_csv("tests/data/malformed.csv", None, Some(true)); + assert!(result.is_ok(), "Should succeed with error recovery enabled"); + let table = result.unwrap(); + + // In recovery mode we should get all rows that can be recovered + // The row with the unclosed quote will be skipped + // Row with missing field should be padded with a null + // Row with too many fields should be truncated + + // Check row count + assert!(table.rows().len() >= 3, "Should have at least 3 valid rows"); + + // Check specific row data for the rows we know should be there + let rows = table.rows(); + assert_eq!( + rows[0][1].to_string(), + "Alice", + "First row should have Alice" ); - // Read rows with enhanced error handling - let should_recover = recover_errors.unwrap_or(false); - let mut skipped_rows = 0; - let mut row_number = 0; - - // Reusable row buffer to avoid per-row allocations - let column_count = table.column_count(); - let mut row_buffer = Vec::with_capacity(column_count); - - for result in csv_reader.records() { - row_number += 1; - - match result { - Ok(record) => { - // Clear buffer for reuse - row_buffer.clear(); - - if should_recover && record.len() != column_count { - // In recovery mode, handle rows with different column counts - for i in 0..column_count { - if i < record.len() { - // Use the value if it exists - row_buffer.push(Value::from(record.get(i).unwrap_or(""))); - } else { - // Pad with null values if we need more - row_buffer.push(Value::Null); - } - } - - // Now we have a properly sized row, add it by cloning from buffer - table.add_row_from_slice(&row_buffer)?; - } else { - // Normal path - convert record to a row of values and validate - for field in record.iter() { - row_buffer.push(Value::from(field)); - } - // This call can fail if the columns don't match and we're not in recovery mode - if let Err(e) = table.add_row_from_slice(&row_buffer) { - if should_recover { - // If we're in recovery mode, log and continue - skipped_rows += 1; - eprintln!( - "Warning: Skipping row at line {} with inconsistent field count: {}", - row_number + 1, // +1 for header row - e - ); - } else { - // If we're not in recovery mode, propagate the error - return Err(e); - } - } - } - } - Err(csv_err) if should_recover => { - // Skip this row and continue processing if recovery is enabled - skipped_rows += 1; - eprintln!( - "Warning: Skipping malformed row at line {}: {}", - row_number + 1, // +1 for header row - csv_err - ); - } - Err(csv_err) => { - // Provide detailed error context when failing - return Err(SqawkError::CsvParseError { - file: file_path.to_string_lossy().to_string(), - line: row_number + 1, // +1 for header row - error: format!("{}", csv_err), - }); + // Due to the flexible recovery, exact row positions may vary based on implementation + // Instead, we'll check that certain expected values exist in the table + // Alice, Bob, and Charlie should all be present + + let mut has_alice = false; + let mut _has_bob = false; // Renamed to _has_bob since it's not asserted + let mut has_charlie = false; + + for row in table.rows() { + if row.len() > 1 { + let name = row[1].to_string(); + if name == "Alice" { + has_alice = true; + } else if name == "Bob with an unclosed quote\"" { + _has_bob = true; + } else if name == "Charlie" { + has_charlie = true; } } } - // Report the number of skipped rows if there were any and we're in recovery mode - if should_recover && skipped_rows > 0 { - eprintln!( - "Note: Skipped {} malformed rows while loading {}", - skipped_rows, - file_path.to_string_lossy() - ); + assert!(has_alice, "Table should contain Alice"); + assert!(has_charlie, "Table should contain Charlie"); + } + + #[test] + fn test_malformed_csv_without_recovery() { + // Create the CSV handler + let csv_handler = CsvHandler::new(); + + // Try to load the malformed CSV file without error recovery + let result = csv_handler.load_csv("tests/data/malformed.csv", None, None); + + // Verify that we get an error + assert!(result.is_err()); + + // Check that the error contains CSV error information + // Type of error may vary by implementation (CsvParseError or InvalidSqlQuery) + let err = result.unwrap_err(); + match err { + SqawkError::CsvParseError { file, line, .. } => { + assert!(file.contains("malformed.csv")); + assert!(line > 0); + } + SqawkError::InvalidSqlQuery(msg) => { + assert!(msg.contains("columns")); + } + _ => panic!("Unexpected error type: {:?}", err), } + } - Ok(table) + #[test] + fn test_csv_with_custom_columns() { + // Create the CSV handler + let csv_handler = CsvHandler::new(); + + // Define custom column names + let custom_columns = Some(vec![ + "user_id".to_string(), + "full_name".to_string(), + "user_age".to_string(), + ]); + + // Load the commented CSV file with custom columns + let table = csv_handler + .load_with_options( + "tests/data/commented.csv", + &commented_options(custom_columns), + ) + .unwrap(); + + // Verify the column names were properly set + let columns = table.columns(); + assert_eq!(columns[0], "user_id"); + assert_eq!(columns[1], "full_name"); + assert_eq!(columns[2], "user_age"); + + // Custom columns mean the file has no header row, so its first + // non-comment line ("id,name,age") is data: 4 rows, not 3. + assert_eq!(table.rows().len(), 4); + assert_eq!(table.rows()[0][1].to_string(), "name"); + assert_eq!(table.rows()[1][1].to_string(), "Alice"); } - /// Parse a file specification into table name and file path - /// - /// Handles two formats: - /// 1. `table_name=file_path.csv` - Explicit table name and file path - /// 2. `file_path.csv` - Table name derived from file name - /// - /// # Arguments - /// * `file_spec` - File specification in one of the supported formats - /// - /// # Returns - /// * `Ok((String, PathBuf))` - Tuple of (table_name, file_path) - /// * `Err` - If the file specification is invalid - fn parse_file_spec(&self, file_spec: &str) -> SqawkResult<(String, PathBuf)> { - if let Some((table_name, file_path)) = file_spec.split_once('=') { - // Table name specified explicitly - Ok((table_name.to_string(), PathBuf::from(file_path))) - } else { - // Table name derived from file name - let path = PathBuf::from(file_spec); - // Check that path has a filename - path.file_name().ok_or_else(|| { - SqawkError::InvalidFileSpec(format!("Invalid file specification: {}", file_spec)) - })?; - - let stem = path.file_stem().ok_or_else(|| { - SqawkError::InvalidFileSpec(format!("Invalid file specification: {}", file_spec)) - })?; - - Ok((stem.to_string_lossy().to_string(), path)) - } + #[test] + fn test_csv_hash_lines_are_data_by_default() { + // Without comment skipping, the first line is the header even though it + // starts with '#', and '#' lines are ordinary records. + let table = CsvHandler::new() + .load_csv("tests/data/commented.csv", None, Some(true)) + .unwrap(); + assert_eq!(table.columns(), vec!["# This is a header comment"]); } } diff --git a/src/database.rs b/src/database.rs index 6cbb577..2bb8b03 100644 --- a/src/database.rs +++ b/src/database.rs @@ -157,7 +157,8 @@ impl Database { // Create a new table with the specified columns // Note: No file_path is assigned here as this is a table definition only let columns_len = columns.len(); // Store the length before we move columns - let table = Table::new(table_name, columns, None); + let mut table = Table::new(table_name, columns, None); + table.set_placeholder(true); // Add the table to the database, overwriting any existing definition with the same name // This ensures CLI definitions take precedence @@ -170,7 +171,7 @@ impl Database { self.add_table(table_name.to_string(), table)?; if config.verbose() { - println!( + eprintln!( "Compiled table definition for '{}' with {} columns", table_name, columns_len ); diff --git a/src/delim_handler.rs b/src/delim_handler.rs deleted file mode 100644 index d28df44..0000000 --- a/src/delim_handler.rs +++ /dev/null @@ -1,266 +0,0 @@ -//! Delimiter-separated values file handling module for sqawk -//! -//! This module handles loading and saving delimiter-separated files into in-memory tables. -//! It works with files that use custom delimiters like tabs, colons, or any other character -//! to separate fields, similar to awk's -F option. -//! -//! The module provides functionality for: -//! -//! - Loading files with custom field separators specified by the user -//! - Parsing file specifications in the format [table_name=]file_path -//! - Converting between delimited records and the internal Value type -//! - Writing modified tables back to delimiter-separated files -//! -//! This implementation reuses the CSV crate's functionality but configures it -//! to use the specified delimiter instead of commas. - -use std::fs::File; -use std::io::{BufReader, BufWriter, Write}; -use std::path::{Path, PathBuf}; - -use tempfile::NamedTempFile; - -use crate::capacity::{generate_alpha_columns, is_likely_data_row}; -use crate::error::{SqawkError, SqawkResult}; -use crate::table::{Table, Value}; - -/// Handles loading and saving delimiter-separated value files -/// -/// This struct provides methods for working with files that use custom delimiters -/// (like tabs, pipes, or spaces) instead of commas. It's similar to the CsvHandler -/// but allows for flexibility in the field separator character. -/// -/// Key capabilities: -/// - Loading files with custom field separators -/// - Inferring column names when headers are missing -/// - Automatic data type detection for values -/// - Writing tables back to delimited files -pub struct DelimHandler {} - -impl Default for DelimHandler { - fn default() -> Self { - Self::new() - } -} - -impl DelimHandler { - /// Create a new DelimHandler - /// - /// # Returns - /// A new DelimHandler instance ready to load and manage delimiter-separated files - pub fn new() -> Self { - DelimHandler {} - } - - /// Save a table to a delimiter-separated file - /// - /// Uses atomic write pattern: writes to a temporary file first, then renames - /// into place. This preserves the original file if writing fails midway. - /// - /// # Arguments - /// * `table` - The table to save - /// * `file_path` - The path to the file - /// * `delimiter` - The delimiter to use (e.g., ",", "\t") - /// - /// # Returns - /// * `SqawkResult<()>` - Result of the operation - pub fn save_delimited( - &self, - table: &Table, - file_path: &Path, - delimiter: &str, - ) -> SqawkResult<()> { - // Create temp file in the same directory as target (required for atomic rename) - let parent = file_path.parent().unwrap_or(Path::new(".")); - let temp_file = NamedTempFile::new_in(parent).map_err(SqawkError::IoError)?; - - // Write data in a scope so the writer is dropped before we call persist - { - // Create a buffered writer for the temp file - let mut writer = BufWriter::new(temp_file.as_file()); - - // Write the header row - let header = table.columns().join(delimiter); - writeln!(writer, "{}", header).map_err(SqawkError::IoError)?; - - // Write data rows - for row in table.rows() { - let row_values: Vec = row.iter().map(|value| value.to_string()).collect(); - - let row_str = row_values.join(delimiter); - writeln!(writer, "{}", row_str).map_err(SqawkError::IoError)?; - } - - // Flush the buffered writer before it's dropped - writer.flush().map_err(SqawkError::IoError)?; - } - - // Sync to disk before rename to ensure data durability - temp_file - .as_file() - .sync_all() - .map_err(SqawkError::IoError)?; - - // Atomically rename temp file to target path - temp_file - .persist(file_path) - .map_err(|e| SqawkError::IoError(e.error))?; - - Ok(()) - } - - /// Load a delimiter-separated file into an in-memory table - /// - /// This method parses files with the specified delimiter and header rows, - /// creating tables with appropriate column names and automatically inferring - /// data types for each cell. - /// - /// # Headers and Column Detection - /// The method implements several strategies for determining column names: - /// - Uses custom column names if provided - /// - Detects header row in the file when present - /// - Intelligently determines if first row is data rather than headers (for files like /etc/passwd) - /// - Generates alphabetical column names (a, b, c...) when headers are missing - /// - /// # Special Delimiter Handling - /// - Supports single-character delimiters (e.g., comma, pipe, colon) - /// - Special handling for tab delimiter with `\t` notation - /// - Supports comment lines starting with '#' for system files - /// - /// # Arguments - /// * `file_spec` - File specification in the format `[table_name=]file_path` - /// * `delimiter` - Delimiter character to use for parsing - /// * `custom_columns` - Optional custom column names to use instead of detected/generated ones - /// - /// # Returns - /// * `Ok(Table)` - The successfully loaded table - /// * `Err` if there was an error parsing the file spec, opening the file, or parsing the file data - pub fn load_delimited( - &self, - file_spec: &str, - delimiter: &str, - custom_columns: Option>, - ) -> SqawkResult
{ - // Parse file spec to get table name and file path - let (table_name, file_path) = self.parse_file_spec(file_spec)?; - - // Open the file - let file = File::open(&file_path)?; - let reader = BufReader::new(file); - - // Get the delimiter as a byte - let delimiter_byte = if delimiter.len() == 1 { - delimiter.as_bytes()[0] - } else if delimiter == "\\t" { - b'\t' // Handle special case for tab - } else { - return Err(SqawkError::InvalidFileSpec(format!( - "Invalid delimiter: {}. Must be a single character.", - delimiter - ))); - }; - - // has_headers is true by default - first row is column names - let has_headers = true; - - // Create a CSV reader with custom delimiter - // Also add support for comment lines (starting with #) for system files like /etc/passwd - let mut csv_reader = csv::ReaderBuilder::new() - .has_headers(has_headers) - .delimiter(delimiter_byte) - .comment(Some(b'#')) // Support for comment lines starting with # - .flexible(true) // Allow for variable number of fields - .from_reader(reader); - - // If custom column names are provided, use them - // Otherwise detect/generate headers based on file content - let headers = if let Some(columns) = custom_columns { - // Use the provided custom column names (first row will be treated as data) - columns - } else { - // No custom column names, generate or detect from file - match csv_reader.headers().map_err(SqawkError::CsvError) { - Ok(header_row) => { - let fields: Vec<&str> = header_row.iter().collect(); - if is_likely_data_row(&fields) { - // First row is data, generate a,b,c column names - generate_alpha_columns(fields.len()) - } else { - // Use the headers as they are - fields.iter().map(|s| s.to_string()).collect() - } - } - Err(_) => { - // If we couldn't read headers, try to determine column count from first record - let record_iter = csv_reader.records(); - let first_record = record_iter.into_iter().next(); - - if let Some(Ok(record)) = first_record { - generate_alpha_columns(record.len()) - } else { - // Fallback to a minimal set if we can't determine field count - vec!["a".to_string()] - } - } - } - }; - - // Create a new table with the determined headers and custom delimiter - let mut table = Table::new_with_delimiter( - &table_name, - headers, - Some(file_path.clone()), - delimiter.to_string(), - ); - - // Read rows with reusable buffer to avoid per-row allocations - let column_count = table.column_count(); - let mut row_buffer = Vec::with_capacity(column_count); - - for result in csv_reader.records() { - let record = result.map_err(SqawkError::CsvError)?; - - // Clear buffer for reuse and fill with values - row_buffer.clear(); - for field in record.iter() { - row_buffer.push(Value::from(field)); - } - - table.add_row_from_slice(&row_buffer)?; - } - - Ok(table) - } - - /// Parse a file specification into table name and file path - /// - /// Handles two formats: - /// 1. `table_name=file_path` - Explicit table name and file path - /// 2. `file_path` - Table name derived from file name - /// - /// # Arguments - /// * `file_spec` - File specification in one of the supported formats - /// - /// # Returns - /// * `Ok((String, PathBuf))` - Tuple of (table_name, file_path) - /// * `Err` - If the file specification is invalid - fn parse_file_spec(&self, file_spec: &str) -> SqawkResult<(String, PathBuf)> { - if let Some((table_name, file_path)) = file_spec.split_once('=') { - // Table name specified explicitly - Ok((table_name.to_string(), PathBuf::from(file_path))) - } else { - // Table name derived from file name - let path = PathBuf::from(file_spec); - // Check that path has a filename - path.file_name().ok_or_else(|| { - SqawkError::InvalidFileSpec(format!("Invalid file specification: {}", file_spec)) - })?; - - let stem = path.file_stem().ok_or_else(|| { - SqawkError::InvalidFileSpec(format!("Invalid file specification: {}", file_spec)) - })?; - - Ok((stem.to_string_lossy().to_string(), path)) - } - } -} diff --git a/src/file_handler.rs b/src/file_handler.rs index 96e16b4..b1daf1e 100644 --- a/src/file_handler.rs +++ b/src/file_handler.rs @@ -10,13 +10,10 @@ use std::path::{Path, PathBuf}; use crate::config::AppConfig; -use crate::csv_handler::CsvHandler; +use crate::csv_handler::{self, HeaderMode, ReadOptions}; use crate::database::Database; -use crate::delim_handler::DelimHandler; use crate::error::{SqawkError, SqawkResult}; -use crate::storage::mmap::MmapStorage; -use crate::storage::Storage; -use crate::table::Table; +use crate::table::{delimiter_byte, Table}; /// File operand that names standard input rather than a path on disk. pub const STDIN_SPEC: &str = "-"; @@ -45,12 +42,6 @@ pub enum FileFormat { /// - Providing access to the database for table operations /// - Handling table loading and saving with proper format detection pub struct FileHandler { - /// Handler for CSV files (comma-separated values) - csv_handler: CsvHandler, - - /// Handler for delimiter-separated files (tab, pipe, etc.) - delim_handler: DelimHandler, - /// Reference to a database object which is the source of truth for tables /// Stored as a raw pointer to avoid borrowing limitations database: *mut Database, @@ -58,13 +49,9 @@ pub struct FileHandler { /// Application configuration for global settings like verbosity and field separators config: AppConfig, - /// Standard input, spooled to a temporary file. - /// - /// Stdin is not seekable and cannot be mmap'd, so a `-` operand is copied - /// to a temp file and then loaded by the ordinary path. The handle is kept - /// here because dropping it deletes the file, which an mmap'd table is - /// still reading from. At most one `-` operand is accepted per run. - stdin_spool: Option, + /// Whether standard input has already been read as a table. At most one + /// `-` operand is accepted per run. + stdin_consumed: bool, /// Name of the table loaded from `-`, if any. Only used to explain why /// `--write` cannot save it. @@ -90,12 +77,10 @@ impl FileHandler { /// A new FileHandler instance ready to load and manage tables pub fn new(config: &AppConfig, database: &mut Database) -> Self { FileHandler { - csv_handler: CsvHandler::new(), - delim_handler: DelimHandler::new(), // SAFETY: The caller must ensure that the database outlives this FileHandler database: database as *mut Database, config: config.clone(), - stdin_spool: None, + stdin_consumed: false, stdin_table: None, stdin_available: true, } @@ -112,37 +97,23 @@ impl FileHandler { self.stdin_available = false; } - /// Copy standard input to a temporary file and return its path. - /// - /// The `.csv` suffix is deliberate: it makes format detection treat piped - /// input as comma-separated, which matches what sqawk itself writes to - /// stdout, so `sqawk ... | sqawk ... -` round-trips. `-F` still overrides. - fn spool_stdin(&mut self) -> SqawkResult { + /// Read all of standard input. + fn read_stdin(&mut self) -> SqawkResult> { if !self.stdin_available { return Err(SqawkError::InvalidFileSpec( "cannot read standard input here: the REPL reads its own commands from stdin" .to_string(), )); } - if self.stdin_spool.is_some() { + if self.stdin_consumed { return Err(SqawkError::InvalidFileSpec( "standard input can only be read once; '-' was given more than once".to_string(), )); } - - let mut spool = tempfile::Builder::new() - .prefix("sqawk-stdin-") - .suffix(".csv") - .tempfile()?; - // No sync_all: this file is deleted at exit, io::copy writes straight - // to the File with no user-space buffer, and the mmap below reads - // through the page cache. Forcing durability would just add a full - // flush of the whole pipe before the first query runs. - std::io::copy(&mut std::io::stdin().lock(), spool.as_file_mut())?; - - let path = spool.path().to_path_buf(); - self.stdin_spool = Some(spool); - Ok(path) + self.stdin_consumed = true; + let mut data = Vec::new(); + std::io::Read::read_to_end(&mut std::io::stdin().lock(), &mut data)?; + Ok(data) } /// Get a mutable reference to the database @@ -183,114 +154,95 @@ impl FileHandler { /// # Returns /// * `SqawkResult>` - Tuple of (table_name, file_path) if successful pub fn load_file(&mut self, file_spec: &str) -> SqawkResult> { - // Parse file spec to get table name and file path let (table_name, file_path) = self.parse_file_spec(file_spec)?; - // A "-" operand is replaced by the temp file stdin was spooled into, - // so everything below loads it as an ordinary file. `from_stdin` then - // detaches that path again, because the temp file is not somewhere - // --write may ever write back to. + // A "-" operand reads standard input. It is parsed as CSV (comma, + // header row) unless -F says otherwise, which matches what sqawk + // itself writes to stdout, so `sqawk ... | sqawk ... -` round-trips. + // It has no writeback target: --write must never touch it. let from_stdin = file_path == Path::new(STDIN_SPEC); - let (file_spec, file_path) = if from_stdin { - let spooled = self.spool_stdin()?; - ( - format!("{}={}", table_name, spooled.to_string_lossy()), - spooled, - ) - } else { - (file_spec.to_string(), file_path) - }; - let file_spec = file_spec.as_str(); - let file_path_str = if from_stdin { STDIN_SPEC.to_string() } else { file_path.to_string_lossy().to_string() }; - // First, check if the table already exists in the database - // This could happen if it was defined through CLI table definitions (--tabledef) - let predefined_columns: Option>; - { - // Create a temporary scope for the database borrow + // A table that already exists is either a --tabledef placeholder, + // whose column names this file supplies the data for, or a genuine + // table, in which case two inputs resolve to the same name. + let predefined_columns: Option> = { let db = self.database_mut(); - predefined_columns = if db.has_table(&table_name) { - // Get the predefined column names to use instead of auto-detected ones - db.get_table(&table_name).ok().map(|t| t.columns().to_vec()) - } else { - None - }; - } + match db.get_table(&table_name) { + Ok(t) if t.is_placeholder() => Some(t.columns().to_vec()), + Ok(_) => return Err(SqawkError::TableAlreadyExists(table_name)), + Err(_) => None, + } + }; - // Show verbose output if needed let verbose = self.config.verbose(); if predefined_columns.is_some() && verbose { - println!( + eprintln!( "Table '{}' has predefined schema, using those column names", table_name ); } - // Determine the file format based on extension - let format = self.detect_format(&file_path); - - // Column definitions now come exclusively from Database - // No need for custom columns logic here anymore - - // Determine delimiter for this file - // If -F flag is provided, always use it (overrides format detection) - let delimiter_str = if let Some(sep) = self.config.field_separator() { - sep + // Delimiter: -F if given, else comma for .csv (and stdin and files + // without an extension), tab for anything else. + let format = if from_stdin { + FileFormat::Csv } else { - match format { + self.detect_format(&file_path) + }; + let delimiter_str = self + .config + .field_separator() + .unwrap_or_else(|| match format { FileFormat::Csv => ",".to_string(), FileFormat::Delimited => "\t".to_string(), - } + }); + + // Header policy: a .csv file (or stdin) always has a header row, and + // its '#' lines are data. Other delimited files get the headerless + // guess (for files like /etc/passwd) and '#' comment lines. + let csv_rules = from_stdin || Self::has_csv_extension(&file_path); + let header = if self.config.no_header() { + HeaderMode::Absent + } else if csv_rules { + HeaderMode::Present + } else { + HeaderMode::Guess }; - let delimiter_byte = delimiter_str.as_bytes()[0]; - - // Try to use mmap for on-disk files (zero-copy loading) - // Only use mmap if the file exists and is a regular file - let table = if file_path.is_file() { - // Use memory-mapped storage for zero-copy access - // Pass predefined columns so mmap treats first row as data when appropriate - match MmapStorage::open_with_columns( - &file_path, - delimiter_byte, - predefined_columns.clone(), - ) { - Ok(mmap_storage) => { - if verbose { - println!("Using mmap storage for table '{}'", table_name); - } - let columns = mmap_storage.columns().to_vec(); - Table::with_storage( - &table_name, - columns, - Some(file_path.clone()), - delimiter_str, - Storage::Mmap(mmap_storage), - ) - } - Err(e) => { - // Fall back to regular loading if mmap fails - if verbose { - println!( - "Mmap failed for '{}', falling back to regular loading: {}", - table_name, e - ); - } - self.load_file_regular(file_spec, &format, &predefined_columns)? - } - } + let opts = ReadOptions { + delimiter: delimiter_byte(&delimiter_str), + header, + comments: !csv_rules, + columns: predefined_columns.clone(), + recover_errors: false, + }; + + let data = if from_stdin { + self.read_stdin()? + } else { + std::fs::read(&file_path)? + }; + let target = if from_stdin { + None } else { - // Stdin or other non-file source - use regular loading - self.load_file_regular(file_spec, &format, &predefined_columns)? + Some(file_path.clone()) }; + let table = csv_handler::read_table( + &table_name, + &data, + target, + delimiter_str, + &opts, + &file_path_str, + )?; // A source with no header row yields a table with no columns, which - // the compiler cannot project from -- it indexed result_regs[0] and - // panicked. Empty input is routine for a pipe, so say what is wrong. + // the compiler cannot project from. Empty input is routine for a + // pipe, so say what is wrong. if table.columns().is_empty() { return Err(SqawkError::InvalidFileSpec(format!( "'{}' has no header row: expected a first line of column names", @@ -298,63 +250,30 @@ impl FileHandler { ))); } - // Drop the spool path so the table has no writeback target: --write - // must never rewrite the temp file standing in for stdin. - let mut table = table; if from_stdin { - table.detach_file_path(); self.stdin_table = Some(table_name.clone()); } - // Now that we have the table, we can update the database without borrowing conflicts - { - // Create a new scope for database operations - let db = self.database_mut(); - - // Handle existing schema if needed (from --tabledef) - if predefined_columns.is_some() { - // Remove the placeholder table created by --tabledef - // We're replacing it with the actual table containing data - if verbose { - println!( - "Replacing predefined schema for '{}' with loaded data", - table_name - ); - } - db.remove_table(&table_name); + let db = self.database_mut(); + if predefined_columns.is_some() { + // Replace the --tabledef placeholder with the loaded table. + if verbose { + eprintln!( + "Replacing predefined schema for '{}' with loaded data", + table_name + ); } - - // Add the table to the database - db.add_table(table_name.clone(), table)?; + db.remove_table(&table_name); } + db.add_table(table_name.clone(), table)?; Ok(Some((table_name, file_path_str))) } - /// Load a file using the regular (non-mmap) method - /// - /// This is the fallback method used when mmap is not available or fails. - fn load_file_regular( - &self, - file_spec: &str, - format: &FileFormat, - predefined_columns: &Option>, - ) -> SqawkResult
{ - match format { - FileFormat::Csv => { - // Load the table from the CSV file - self.csv_handler - .load_csv(file_spec, predefined_columns.clone(), None) - } - FileFormat::Delimited => { - let delimiter = self - .config - .field_separator() - .unwrap_or_else(|| "\t".to_string()); - self.delim_handler - .load_delimited(file_spec, &delimiter, predefined_columns.clone()) - } - } + /// Whether `path` has a `.csv` extension (any case). + fn has_csv_extension(path: &Path) -> bool { + path.extension() + .is_some_and(|ext| ext.to_string_lossy().eq_ignore_ascii_case("csv")) } /// Parse a file specification into a table name and path @@ -537,37 +456,12 @@ impl FileHandler { // For tables created with CREATE TABLE, the file may not exist yet // Make sure parent directories exist if let Some(parent) = file_path.parent() { - if !parent.exists() { + if !parent.as_os_str().is_empty() && !parent.exists() { std::fs::create_dir_all(parent).map_err(SqawkError::IoError)?; } } - // Determine the format based on the file extension - let format = self.detect_format(file_path); - - // Get the delimiter from the table - let delimiter = table.delimiter(); - - // Save the table based on the format - match format { - FileFormat::Csv => { - // Delegation to CSV handler (comma is the standard CSV delimiter) - if delimiter == "," { - self.csv_handler.save_csv(table, file_path)?; - } else { - // If delimiter is not a comma, use the delimited handler - self.delim_handler - .save_delimited(table, file_path, delimiter)?; - } - } - FileFormat::Delimited => { - // Delegation to delimited handler - self.delim_handler - .save_delimited(table, file_path, delimiter)?; - } - } - - Ok(()) + csv_handler::save_table(table, file_path) } /// Detect file format based on file extension diff --git a/src/lib.rs b/src/lib.rs deleted file mode 100644 index 6c4d398..0000000 --- a/src/lib.rs +++ /dev/null @@ -1,19 +0,0 @@ -//! sqawk library -//! -//! This crate provides a library for the sqawk CLI utility. -//! It facilitates SQL querying against in-memory CSV tables. - -pub mod aggregate; -pub mod capacity; -pub mod cli; -pub mod config; -pub mod csv_handler; -pub mod database; -pub mod delim_handler; -pub mod error; -pub mod file_handler; -pub mod repl; -pub mod sql_executor; -pub mod storage; -pub mod table; -pub mod vm; diff --git a/src/main.rs b/src/main.rs index 2baf0bf..948c299 100644 --- a/src/main.rs +++ b/src/main.rs @@ -32,12 +32,10 @@ mod cli; mod config; mod csv_handler; mod database; -mod delim_handler; mod error; mod file_handler; mod repl; mod sql_executor; -mod storage; mod table; mod vm; @@ -72,18 +70,24 @@ fn main() -> Result<()> { // Step 1b: Create a centralized application configuration // This will be passed to all components that need configuration settings + let field_separator = args + .field_separator + .as_deref() + .map(cli::validate_separator) + .transpose()?; let config = AppConfig::new( - args.verbose, // Verbose output flag - args.field_separator.clone(), // Field separator for tables - args.tabledef.clone(), // Table column definitions - args.write, // Whether to write changes to files - ); + args.verbose, // Verbose output flag + field_separator, // Field separator for tables + args.tabledef.clone(), // Table column definitions + args.write, // Whether to write changes to files + ) + .with_no_header(args.no_header); // Configure diagnostics output if verbose mode is enabled (-v flag) // This is important for debugging and understanding the execution flow if config.verbose() { - println!("Running in verbose mode"); - println!("Arguments: {args:?}"); + eprintln!("Running in verbose mode"); + eprintln!("Arguments: {args:?}"); } // Step 2a: Create a new Database instance to serve as the central store for tables @@ -125,9 +129,9 @@ fn main() -> Result<()> { // Log table loading results in verbose mode if config.verbose() { let table_count = file_handler.table_count(); - println!("Loaded {table_count} tables"); + eprintln!("Loaded {table_count} tables"); for table_name in file_handler.table_names() { - println!("Table '{table_name}' loaded"); + eprintln!("Table '{table_name}' loaded"); } } @@ -151,6 +155,8 @@ fn main() -> Result<()> { } } + let mut stdout_open = true; + // Process each SQL statement in the order specified on the command line // This allows operations like: UPDATE -> DELETE -> SELECT to see the effects for sql in &args.sql { @@ -168,13 +174,22 @@ fn main() -> Result<()> { for table in &result { if config.verbose() { let row_count = table.row_count(); - println!("Query returned {row_count} rows"); + eprintln!("Query returned {row_count} rows"); + } + // Print the result table in delimiter-separated format. Once the + // reader of a pipe has gone away (`sqawk ... | head`), stop + // printing but keep going: later statements and --write still run. + if stdout_open { + if let Err(e) = table.print_to_stdout() { + if e.kind() != std::io::ErrorKind::BrokenPipe { + return Err(e).context("Failed to write results"); + } + stdout_open = false; + } } - // Print the result table in delimiter-separated format - table.print_to_stdout()?; } if result.is_empty() && config.verbose() { - println!("Query executed successfully (no results to display)"); + eprintln!("Query executed successfully (no results to display)"); } } @@ -188,8 +203,14 @@ fn main() -> Result<()> { .context("Failed to save modified tables")?; } else if config.verbose() { // In verbose mode, remind the user that changes weren't saved - println!("Changes not saved: use --write to save changes to files"); + eprintln!("Changes not saved: use --write to save changes to files"); } + // The process is about to exit, so freeing every loaded cell one by one + // is wasted work (measurable on large inputs); let the OS reclaim it. + drop(sql_executor); + drop(file_handler); + std::mem::forget(database); + Ok(()) } diff --git a/src/repl.rs b/src/repl.rs index d3defc4..051ebb2 100644 --- a/src/repl.rs +++ b/src/repl.rs @@ -441,12 +441,18 @@ impl<'a> Repl<'a> { println!("Query returned no rows"); } else { println!("Query returned {} rows", result_set.rows.len()); - // Print column headers - println!("{}", result_set.columns.join(",")); - - // Print rows + // Header and rows as CSV, quoted where a field needs it + // (NULL is already an empty field in a ResultSet). + let writer = crate::table::RecordWriter { + delimiter: b',', + terminator: b"\n", + comments: false, + }; + let stdout = std::io::stdout(); + let mut out = stdout.lock(); + let _ = writer.write(&mut out, result_set.columns.iter().map(String::as_str)); for row in &result_set.rows { - println!("{}", row.join(",")); + let _ = writer.write(&mut out, row.iter().map(String::as_str)); } } } diff --git a/src/sql_executor.rs b/src/sql_executor.rs index 478e479..b54dda5 100644 --- a/src/sql_executor.rs +++ b/src/sql_executor.rs @@ -77,7 +77,7 @@ impl<'a> SqlExecutor<'a> { /// deliberately NOT merged, since each has its own column schema. pub fn execute(&mut self, sql: &str) -> SqawkResult> { if self.config.verbose() { - println!("Executing SQL: {}", sql); + eprintln!("Executing SQL: {}", sql); } // Execute via VM engine diff --git a/src/storage/memory.rs b/src/storage/memory.rs deleted file mode 100644 index d1adaf2..0000000 --- a/src/storage/memory.rs +++ /dev/null @@ -1,60 +0,0 @@ -//! In-memory storage backend -//! -//! This module provides traditional in-memory storage for table rows. -//! All data is heap-allocated and owned by the storage. - -use crate::table::Row; - -/// In-memory storage backend -/// -/// Stores table rows in a heap-allocated vector. This is the traditional -/// storage mode where all data is copied into memory during loading. -#[derive(Debug, Clone)] -pub struct MemoryStorage { - /// Row data - rows: Vec, -} - -impl MemoryStorage { - /// Create a new empty memory storage - pub fn new() -> Self { - Self { rows: Vec::new() } - } - - /// Create memory storage with existing rows - #[allow(dead_code)] - pub fn with_rows(rows: Vec) -> Self { - Self { rows } - } - - /// Get the number of rows - pub fn row_count(&self) -> usize { - self.rows.len() - } - - /// Get all rows as a slice - pub fn rows(&self) -> &[Row] { - &self.rows - } - - /// Get mutable reference to rows - pub fn rows_mut(&mut self) -> &mut Vec { - &mut self.rows - } - - /// Add a row to the storage - pub fn push_row(&mut self, row: Row) { - self.rows.push(row); - } - - /// Replace all rows - pub fn replace_rows(&mut self, rows: Vec) { - self.rows = rows; - } -} - -impl Default for MemoryStorage { - fn default() -> Self { - Self::new() - } -} diff --git a/src/storage/mmap.rs b/src/storage/mmap.rs deleted file mode 100644 index 779e15f..0000000 --- a/src/storage/mmap.rs +++ /dev/null @@ -1,366 +0,0 @@ -//! Memory-mapped storage backend -//! -//! This module provides zero-copy access to CSV files by memory-mapping them. -//! String values reference data directly in the mmap'd region instead of -//! being copied to the heap. - -use std::borrow::Cow; -use std::fs::File; -use std::path::Path; - -use memmap2::Mmap; - -use crate::capacity::{ - estimate_row_count, generate_alpha_columns, is_comment_line, is_likely_data_row, - DEFAULT_ROW_CAPACITY, -}; -use crate::error::{SqawkError, SqawkResult}; -use crate::table::{Row, Value}; - -/// Memory-mapped storage backend -/// -/// This storage backend maps a CSV file directly into memory and parses -/// rows lazily. String values borrow directly from the mmap'd region, -/// avoiding heap allocation. -/// -/// # Safety -/// -/// This struct uses unsafe code to create `'static` references from the -/// mmap'd region. This is safe because: -/// 1. The Mmap is owned by this struct -/// 2. The rows reference data within the Mmap -/// 3. The struct cannot be mutated after creation -/// 4. When the struct is dropped, both the rows and Mmap are dropped together -#[derive(Debug)] -pub struct MmapStorage { - /// The memory-mapped file data - #[allow(dead_code)] - mmap: Mmap, - /// Column names from the header row - columns: Vec, - /// Parsed row data (borrows from mmap) - rows: Vec, -} - -#[allow(dead_code)] -impl MmapStorage { - /// Open a CSV file with memory-mapping - /// - /// # Arguments - /// * `path` - Path to the CSV file - /// * `delimiter` - Field delimiter (e.g., ',' for CSV, '\t' for TSV) - /// - /// # Returns - /// * `Ok(MmapStorage)` if the file was successfully opened and parsed - /// * `Err` if the file couldn't be opened or parsed - pub fn open>(path: P, delimiter: u8) -> SqawkResult { - Self::open_with_columns(path, delimiter, None) - } - - /// Open with predefined column names (for --tabledef support) - /// - /// When predefined columns are provided, the first row is treated as data, - /// not headers. - pub fn open_with_columns>( - path: P, - delimiter: u8, - predefined_columns: Option>, - ) -> SqawkResult { - let file = File::open(path.as_ref()).map_err(SqawkError::IoError)?; - - // Safety: We're only reading the file, and the Mmap will be kept alive - // for as long as the MmapStorage exists - let mmap = unsafe { Mmap::map(&file).map_err(SqawkError::IoError)? }; - - // Hint to the kernel for better read-ahead - #[cfg(unix)] - { - let ptr = mmap.as_ptr() as *mut libc::c_void; - let len = mmap.len(); - unsafe { - // MADV_SEQUENTIAL: Expect sequential page references - libc::madvise(ptr, len, libc::MADV_SEQUENTIAL); - // MADV_WILLNEED: Will need these pages, initiate read-ahead - libc::madvise(ptr, len, libc::MADV_WILLNEED); - } - } - - // Parse the CSV structure - let (columns, rows) = Self::parse_csv(&mmap, delimiter, predefined_columns)?; - - Ok(Self { - mmap, - columns, - rows, - }) - } - - /// Parse CSV data from memory-mapped region - /// - /// # Arguments - /// * `data` - Memory-mapped file data - /// * `delimiter` - Field delimiter byte - /// * `predefined_columns` - If Some, use these as column names and treat first row as data - /// - /// # Safety - /// This function creates Value::String with Cow::Borrowed references that - /// point into the mmap'd memory. The caller must ensure the returned rows - /// do not outlive the mmap. - fn parse_csv( - data: &Mmap, - delimiter: u8, - predefined_columns: Option>, - ) -> SqawkResult<(Vec, Vec)> { - if data.is_empty() { - return Ok((Vec::new(), Vec::new())); - } - - // Find line boundaries - estimate capacity from file size - let estimated_lines = estimate_row_count(data.len()); - let mut lines: Vec<&[u8]> = Vec::with_capacity(estimated_lines); - let mut start = 0; - - for (i, &byte) in data.iter().enumerate() { - if byte == b'\n' { - // Handle \r\n line endings - let end = if i > 0 && data[i - 1] == b'\r' { - i - 1 - } else { - i - }; - if end > start { - lines.push(&data[start..end]); - } - start = i + 1; - } - } - // Handle last line without newline - if start < data.len() { - let end = if data[data.len() - 1] == b'\r' { - data.len() - 1 - } else { - data.len() - }; - if end > start { - lines.push(&data[start..end]); - } - } - - // Filter out comment lines (starting with #) - lines.retain(|line| !is_comment_line(line)); - - if lines.is_empty() { - return Ok((Vec::new(), Vec::new())); - } - - // Determine columns and where data rows start - let (columns, data_start) = if let Some(cols) = predefined_columns { - // Use predefined columns, all lines are data - (cols, 0) - } else { - // Parse first row and check if it looks like data or headers - let header_line = lines[0]; - let fields: Vec = Self::split_fields(header_line, delimiter) - .iter() - .map(|field| String::from_utf8_lossy(field).into_owned()) - .collect(); - - if is_likely_data_row(&fields) { - // First row is data, generate a,b,c column names - (generate_alpha_columns(fields.len()), 0) - } else { - // First row is headers - (fields, 1) - } - }; - - // Parse data rows - let mut rows = Vec::with_capacity(lines.len() - data_start); - - for line in lines.iter().skip(data_start) { - let fields = Self::split_fields(line, delimiter); - let mut row = Vec::with_capacity(columns.len()); - - for field in fields { - // Convert field bytes to a Value - // Safety: We use unsafe to extend the lifetime to 'static - // This is safe because the mmap outlives the rows (both are in the same struct) - let value = unsafe { Self::parse_field_borrowed(field) }; - row.push(value); - } - - // Pad row with nulls if it has fewer fields than headers - while row.len() < columns.len() { - row.push(Value::Null); - } - - rows.push(row); - } - - Ok((columns, rows)) - } - - /// Split a line into fields by delimiter - fn split_fields(line: &[u8], delimiter: u8) -> Vec<&[u8]> { - let mut fields = Vec::with_capacity(DEFAULT_ROW_CAPACITY); - let mut start = 0; - let mut in_quotes = false; - - for (i, &byte) in line.iter().enumerate() { - if byte == b'"' { - in_quotes = !in_quotes; - } else if byte == delimiter && !in_quotes { - fields.push(&line[start..i]); - start = i + 1; - } - } - - // Add the last field - fields.push(&line[start..]); - - fields - } - - /// Parse a field as a Value with borrowed string data - /// - /// # Safety - /// The returned Value contains a Cow::Borrowed that references the input bytes. - /// The caller must ensure the bytes outlive the Value. - unsafe fn parse_field_borrowed(field: &[u8]) -> Value { - // Convert to str first - let s = match std::str::from_utf8(field) { - Ok(s) => s, - Err(_) => { - // If not valid UTF-8, convert lossily and return owned - return Value::String(Cow::Owned(String::from_utf8_lossy(field).into_owned())); - } - }; - - // Trim whitespace and quotes - let trimmed = s.trim(); - let trimmed = if trimmed.starts_with('"') && trimmed.ends_with('"') && trimmed.len() >= 2 { - &trimmed[1..trimmed.len() - 1] - } else { - trimmed - }; - - // Try to parse as integer - if let Ok(i) = trimmed.parse::() { - return Value::Integer(i); - } - - // Try to parse as float - if let Ok(f) = trimmed.parse::() { - return Value::Float(f); - } - - // Try to parse as boolean (case-insensitive without allocation) - if trimmed.is_empty() { - return Value::Null; - } - if trimmed.eq_ignore_ascii_case("true") || trimmed.eq_ignore_ascii_case("yes") { - return Value::Boolean(true); - } - if trimmed.eq_ignore_ascii_case("false") || trimmed.eq_ignore_ascii_case("no") { - return Value::Boolean(false); - } - - // Return as borrowed string - // Safety: We extend the lifetime to 'static. The caller must ensure the - // data lives long enough. - let static_str: &'static str = std::mem::transmute(trimmed); - Value::String(Cow::Borrowed(static_str)) - } - - /// Get the column names - pub fn columns(&self) -> &[String] { - &self.columns - } - - /// Get the number of rows - pub fn row_count(&self) -> usize { - self.rows.len() - } - - /// Get all rows as a slice - pub fn rows(&self) -> &[Row] { - &self.rows - } -} - -// MmapStorage is not Clone because the mmap'd references would become invalid -// if the Mmap were dropped independently - -#[cfg(test)] -mod tests { - use super::*; - use std::io::Write; - use tempfile::NamedTempFile; - - #[test] - fn test_mmap_storage_basic() { - // Create a temporary CSV file - let mut file = NamedTempFile::new().unwrap(); - writeln!(file, "name,age,city").unwrap(); - writeln!(file, "Alice,30,NYC").unwrap(); - writeln!(file, "Bob,25,LA").unwrap(); - - // Open with mmap - let storage = MmapStorage::open(file.path(), b',').unwrap(); - - // Check columns - assert_eq!(storage.columns(), &["name", "age", "city"]); - - // Check row count - assert_eq!(storage.row_count(), 2); - - // Check first row values - let rows = storage.rows(); - assert_eq!(rows[0].len(), 3); - - // String values should be borrowed - match &rows[0][0] { - Value::String(cow) => { - assert!(matches!(cow, Cow::Borrowed(_))); - assert_eq!(cow.as_ref(), "Alice"); - } - _ => panic!("Expected string value"), - } - - // Integer values should be parsed - assert_eq!(rows[0][1], Value::Integer(30)); - - // Another string value - match &rows[0][2] { - Value::String(cow) => { - assert!(matches!(cow, Cow::Borrowed(_))); - assert_eq!(cow.as_ref(), "NYC"); - } - _ => panic!("Expected string value"), - } - } - - #[test] - fn test_mmap_storage_empty_file() { - let file = NamedTempFile::new().unwrap(); - let storage = MmapStorage::open(file.path(), b',').unwrap(); - - assert!(storage.columns().is_empty()); - assert_eq!(storage.row_count(), 0); - } - - #[test] - fn test_mmap_storage_tsv() { - let mut file = NamedTempFile::new().unwrap(); - writeln!(file, "name\tvalue").unwrap(); - writeln!(file, "test\t42").unwrap(); - - let storage = MmapStorage::open(file.path(), b'\t').unwrap(); - - assert_eq!(storage.columns(), &["name", "value"]); - assert_eq!(storage.row_count(), 1); - - let rows = storage.rows(); - assert_eq!(rows[0][1], Value::Integer(42)); - } -} diff --git a/src/storage/mod.rs b/src/storage/mod.rs deleted file mode 100644 index ca5c70f..0000000 --- a/src/storage/mod.rs +++ /dev/null @@ -1,140 +0,0 @@ -//! Storage backends for table row data -//! -//! This module provides different storage backends for table rows: -//! - `MemoryStorage`: Traditional in-memory storage with owned data -//! - `MmapStorage`: Memory-mapped file storage for zero-copy access - -pub mod memory; -pub mod mmap; - -use crate::table::Row; - -/// Storage backend for table row data -/// -/// This enum wraps different storage implementations, allowing tables -/// to use either in-memory storage or memory-mapped file storage. -/// Column metadata is managed by the Table struct, not the storage. -#[derive(Debug)] -pub enum Storage { - /// In-memory storage with owned data - Memory(memory::MemoryStorage), - /// Memory-mapped file storage (read-only, zero-copy) - #[allow(dead_code)] - Mmap(mmap::MmapStorage), -} - -impl Storage { - /// Create a new empty in-memory storage - pub fn new_memory() -> Self { - Storage::Memory(memory::MemoryStorage::new()) - } - - /// Create in-memory storage with existing rows - #[allow(dead_code)] - pub fn with_rows(rows: Vec) -> Self { - Storage::Memory(memory::MemoryStorage::with_rows(rows)) - } - - /// Get the number of rows - pub fn row_count(&self) -> usize { - match self { - Storage::Memory(m) => m.row_count(), - Storage::Mmap(m) => m.row_count(), - } - } - - /// Get all rows as a slice - pub fn rows(&self) -> &[Row] { - match self { - Storage::Memory(m) => m.rows(), - Storage::Mmap(m) => m.rows(), - } - } - - /// Get a mutable reference to rows (only for mutable backends) - /// - /// Returns None for read-only storage backends like Mmap. - pub fn rows_mut(&mut self) -> Option<&mut Vec> { - match self { - Storage::Memory(m) => Some(m.rows_mut()), - Storage::Mmap(_) => None, // Mmap is read-only - } - } - - /// Add a row to the storage - /// - /// Note: For mmap storage, this will panic. Use is_mutable() to check first. - pub fn push_row(&mut self, row: Row) { - match self { - Storage::Memory(m) => m.push_row(row), - Storage::Mmap(_) => panic!("Cannot add rows to read-only mmap storage"), - } - } - - /// Check if this storage is mutable - #[allow(dead_code)] - pub fn is_mutable(&self) -> bool { - match self { - Storage::Memory(_) => true, - Storage::Mmap(_) => false, - } - } - - /// Replace all rows - /// - /// Note: For mmap storage, this will panic. Use is_mutable() to check first, - /// or call ensure_mutable() before calling this method. - pub fn replace_rows(&mut self, rows: Vec) { - match self { - Storage::Memory(m) => m.replace_rows(rows), - Storage::Mmap(_) => panic!("Cannot replace rows in read-only mmap storage"), - } - } - - /// Convert this storage to a mutable in-memory storage if needed - /// - /// If the storage is already mutable (Memory), this is a no-op. - /// If the storage is read-only (Mmap), it copies all rows to memory, - /// converting borrowed strings to owned strings. - pub fn ensure_mutable(&mut self) { - use crate::table::Value; - use std::borrow::Cow; - - match self { - Storage::Memory(_) => { - // Already mutable, nothing to do - } - Storage::Mmap(mmap) => { - // Copy all rows to memory, converting borrowed strings to owned - let rows: Vec = mmap - .rows() - .iter() - .map(|row| { - row.iter() - .map(|value| match value { - Value::String(cow) => { - // Convert borrowed to owned - Value::String(Cow::Owned(cow.to_string())) - } - // Other value types are Copy or already owned - v => v.clone(), - }) - .collect() - }) - .collect(); - *self = Storage::Memory(memory::MemoryStorage::with_rows(rows)); - } - } - } - - /// Get column names from the storage (for mmap only) - /// - /// Returns None for memory storage (columns are stored in Table). - #[allow(dead_code)] - pub fn columns(&self) -> Option<&[String]> { - match self { - Storage::Memory(_) => None, - Storage::Mmap(m) => Some(m.columns()), - } - } -} diff --git a/src/table.rs b/src/table.rs index 7b57d50..e5d21dd 100644 --- a/src/table.rs +++ b/src/table.rs @@ -10,14 +10,11 @@ //! - Column resolution with qualified names (table.column) use std::borrow::Cow; -use std::collections::HashMap; +use std::collections::{BTreeMap, HashMap}; use std::fmt; use std::path::PathBuf; -use anyhow::Result; - use crate::error::{SqawkError, SqawkResult}; -use crate::storage::Storage; /// Represents a value in a table cell /// @@ -25,9 +22,7 @@ use crate::storage::Storage; /// It supports the common SQL data types and allows for type conversions /// between numeric types (Integer <-> Float) for comparison operations. /// -/// String values use `Cow<'static, str>` to support both: -/// - Owned strings (heap-allocated, from in-memory tables) -/// - Borrowed strings (zero-copy, from memory-mapped files) +/// String values are `Cow<'static, str>`; every stored string is owned. #[derive(Debug, Clone)] pub enum Value { /// Represents a NULL or missing value @@ -83,9 +78,16 @@ impl std::hash::Hash for Value { i.hash(state); } Value::Float(f) => { - // Convert float to bits for hashing to avoid NaN issues - 2_i32.hash(state); - f.to_bits().hash(state); + // A float equal to an integer must hash like that integer, + // because `Integer(1) == Float(1.0)` under PartialEq. + if f.fract() == 0.0 && *f >= i64::MIN as f64 && *f < i64::MAX as f64 { + 1_i32.hash(state); + (*f as i64).hash(state); + } else { + // Bits avoid NaN issues + 2_i32.hash(state); + f.to_bits().hash(state); + } } Value::String(s) => { // Hash the string value @@ -160,65 +162,165 @@ impl fmt::Display for Value { } } -/// Implementation of string conversion to Value with automatic type inference +/// Parse `s` as an integer only if `s` is that integer's canonical spelling. /// -/// This implementation enables automatic type detection when loading data from delimiter-separated files. -/// It attempts to parse the string value in the following order: -/// 1. As an integer (i64) -/// 2. As a floating point number (f64) -/// 3. As a boolean (recognizing various common boolean representations) -/// 4. Empty strings are converted to NULL values -/// 5. Any other content is stored as a string +/// Canonical means `i.to_string() == s`: an optional `-`, no `+`, no leading +/// zeros (`007`), no `-0`, no surrounding whitespace. Checked by hand rather +/// than by formatting, since this runs once per loaded cell. +fn canonical_integer(s: &str) -> Option { + let b = s.as_bytes(); + let digits = match b.first()? { + b'-' => &b[1..], + _ => b, + }; + if digits.is_empty() || !digits.iter().all(u8::is_ascii_digit) { + return None; + } + if digits[0] == b'0' && (digits.len() > 1 || digits.len() != b.len()) { + return None; + } + s.parse::().ok() +} + +/// Parse `s` as a float only if formatting the float back (the same `{}` +/// formatting `Value`'s Display uses) reproduces `s` exactly. /// -/// This type inference approach allows for efficient data storage and comparisons -/// without requiring explicit type declarations in the input files. -impl From<&str> for Value { - fn from(s: &str) -> Self { - // Try to parse as integer first - if let Ok(i) = s.parse::() { - return Value::Integer(i); - } +/// So `1.5` is a float but `1.50`, `1e3`, `2.0`, `.5`, `NaN` and `inf` are +/// not: loading them as numbers would change their spelling on output and on +/// writeback. Display never emits an exponent, so only `-`, digits and `.` +/// can possibly round-trip; anything else is rejected before formatting. +fn canonical_float(s: &str) -> Option { + if s.is_empty() + || !s + .bytes() + .all(|c| c.is_ascii_digit() || c == b'-' || c == b'.') + { + return None; + } + let f = s.parse::().ok()?; + if !f.is_finite() { + return None; + } + if let Some(canonical) = short_decimal_is_canonical(s) { + return canonical.then_some(f); + } + thread_local! { + static BUF: std::cell::RefCell = const { std::cell::RefCell::new(String::new()) }; + } + BUF.with(|buf| { + use std::fmt::Write; + let mut buf = buf.borrow_mut(); + buf.clear(); + let _ = write!(buf, "{}", f); + (*buf == s).then_some(f) + }) +} - // Try to parse as float - if let Ok(fl) = s.parse::() { - return Value::Float(fl); - } +/// Decide canonicity of a plain decimal without formatting, when possible. +/// +/// Display prints the shortest digit string that round-trips, in positional +/// notation. A decimal of at most 15 significant digits round-trips through +/// f64 exactly, and no other decimal that short maps to the same double, so +/// such a string is canonical iff it is in Display's shape: +/// `-?(0|[1-9][0-9]*)\.[0-9]*[1-9]`. Returns None when the fast rule does not +/// apply (more digits, or no point), leaving the caller to format and compare. +fn short_decimal_is_canonical(s: &str) -> Option { + let body = s.strip_prefix('-').unwrap_or(s); + let (int, frac) = body.split_once('.')?; + if int.is_empty() || frac.is_empty() || frac.contains(['.', '-']) || int.contains('-') { + return Some(false); + } + let digits = || int.bytes().chain(frac.bytes()); + let significant = digits().skip_while(|&c| c == b'0').count(); + if significant > 15 { + return None; + } + let shape_ok = (int == "0" || !int.starts_with('0')) && !frac.ends_with('0'); + Some(shape_ok) +} - // Try to parse as boolean - match s.to_lowercase().as_str() { - "true" | "yes" | "1" => return Value::Boolean(true), - "false" | "no" | "0" => return Value::Boolean(false), - "" => return Value::Null, - _ => {} - } +/// Infer the type of a field read from an input file. +/// +/// The rule is chosen so that every loaded cell renders back to exactly the +/// text it was read from -- untouched cells survive `--write` byte for byte: +/// +/// - empty field: NULL (rendered back as an empty field) +/// - canonical integer (`42`, `-7`): Integer +/// - canonical float (`1.5`, `-0.25`): Float +/// - anything else, verbatim and untrimmed: String. That includes `007`, +/// `1.50`, `1e3`, ` 42`, `true`, `NaN`. +/// +/// Numeric-looking strings still behave as numbers where a number is needed +/// (arithmetic, SUM/AVG, comparison with a number, ordering); see the VM. +fn infer_value(s: &str) -> Option { + if s.is_empty() { + return Some(Value::Null); + } + if let Some(i) = canonical_integer(s) { + return Some(Value::Integer(i)); + } + canonical_float(s).map(Value::Float) +} - // Default to owned string - Value::String(Cow::Owned(s.to_string())) +/// Field text to Value, with canonical type inference (see `infer_value`). +impl From<&str> for Value { + fn from(s: &str) -> Self { + infer_value(s).unwrap_or_else(|| Value::String(Cow::Owned(s.to_string()))) } } impl From for Value { fn from(s: String) -> Self { - // Try to parse as integer first - if let Ok(i) = s.parse::() { - return Value::Integer(i); - } + infer_value(&s).unwrap_or(Value::String(Cow::Owned(s))) + } +} - // Try to parse as float - if let Ok(fl) = s.parse::() { - return Value::Float(fl); +impl Value { + /// The text of this value as a delimited-file field. + /// + /// Identical to Display except that NULL is the empty field, which is + /// what NULL loads from. Used for stdout and for writeback. + pub fn field_str(&self) -> Cow<'_, str> { + match self { + Value::Null => Cow::Borrowed(""), + Value::String(s) => Cow::Borrowed(s.as_ref()), + other => Cow::Owned(other.to_string()), } + } - // Try to parse as boolean - match s.to_lowercase().as_str() { - "true" | "yes" | "1" => return Value::Boolean(true), - "false" | "no" | "0" => return Value::Boolean(false), - "" => return Value::Null, - _ => {} + /// This value as a number, if it is one or is text spelling one. + /// + /// Integers stay integers (text such as `007` included); other numeric + /// text becomes a float. Surrounding whitespace is allowed, as awk and + /// SQLite allow it. Returns None for NULL, booleans and non-numeric text. + pub fn as_numeric(&self) -> Option { + match self { + Value::Integer(_) | Value::Float(_) => Some(self.clone()), + Value::String(s) => numeric_text(s), + _ => None, } + } +} - // Default to owned string (avoid extra allocation) - Value::String(Cow::Owned(s)) +/// Interpret text as a number: an i64 if it parses as one, else a finite f64. +pub fn numeric_text(s: &str) -> Option { + let t = s.trim(); + // Cheap rejection first: this runs inside sort comparators. A finite + // number starts with a digit, a sign or a point. + if !t + .bytes() + .next() + .is_some_and(|c| c.is_ascii_digit() || matches!(c, b'-' | b'+' | b'.')) + { + return None; + } + if let Ok(i) = t.parse::() { + return Some(Value::Integer(i)); + } + // `inf` and `NaN` parse but are not finite, so they stay text. + match t.parse::() { + Ok(f) if f.is_finite() => Some(Value::Float(f)), + _ => None, } } @@ -621,7 +723,7 @@ impl<'a> Default for IndexedResultBuilder<'a> { // End Index-Based Result Types // ============================================================================= -/// Represents a table with a storage backend +/// Represents an in-memory table #[derive(Debug)] pub struct Table { /// Name of the table @@ -633,8 +735,8 @@ pub struct Table { /// Map of column names to their indices column_map: HashMap, - /// Row data storage backend (memory or mmap) - storage: Storage, + /// Row data + rows: Vec, /// File path associated with this table (for loading or saving) file_path: Option, @@ -644,6 +746,73 @@ pub struct Table { /// Custom delimiter for this table's file (default is comma) delimiter: String, + + /// How the source file was laid out, so writeback reproduces it. + format: FileLayout, + + /// The source file's `#` comment lines, so writeback reproduces them. + comments: CommentLines, + + /// A `--tabledef` schema with no data yet, waiting for its file + placeholder: bool, +} + +/// Layout details of a table's source file that writeback must reproduce. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct FileLayout { + /// Whether the file has a header line. Headerless files (`--no-header`, + /// `--tabledef`, or a guessed headerless non-.csv file) are written back + /// without one. + pub has_header: bool, + /// Whether records end in CRLF rather than LF. + pub crlf: bool, + /// Whether `#` lines are comments when the file is read. If so, a data + /// row whose first field starts with `#` must be quoted on writeback, or + /// it would be skipped as a comment when read again. + pub comments: bool, +} + +/// The `#` comment lines of a source file read with comment skipping, each +/// kept verbatim (with its own line ending) and anchored to the data around +/// it, so that writeback puts it back where it was. +/// +/// A comment line is anchored to the row that follows it. Rows keep their +/// identity through UPDATE, so their comments stay with them; a deleted row +/// leaves its comments in place, in front of the next surviving row (or of +/// the trailing comments); rows appended by INSERT go after the last row and +/// before the trailing comments. Blank lines are not kept. +#[derive(Debug, Clone, Default, PartialEq, Eq)] +pub struct CommentLines { + /// Lines before the first record (the header, or the first row). + pub head: Vec, + /// Lines before row `i`, by row index. + pub before: BTreeMap>, + /// Lines after the last record. + pub tail: Vec, +} + +impl CommentLines { + /// Move the comments of rows `from..` to the front of the tail, in order: + /// those rows are gone. + fn flush_from(&mut self, from: usize) { + let gone = self.before.split_off(&from); + if gone.is_empty() { + return; + } + let mut lines: Vec = gone.into_values().flatten().collect(); + lines.append(&mut self.tail); + self.tail = lines; + } +} + +impl Default for FileLayout { + fn default() -> Self { + FileLayout { + has_header: true, + crlf: false, + comments: false, + } + } } /// Data type for a column in a table schema @@ -719,10 +888,13 @@ impl Table { name: name.to_string(), cols, column_map, - storage: Storage::new_memory(), + rows: Vec::new(), file_path, modified: false, delimiter: ",".to_string(), // Default to comma delimiter + format: FileLayout::default(), + comments: CommentLines::default(), + placeholder: false, } } @@ -738,44 +910,6 @@ impl Table { table } - /// Create a new table with a specific storage backend - /// - /// This constructor is used for memory-mapped file storage where the - /// storage already contains the parsed data. - pub fn with_storage( - name: &str, - column_names: Vec, - file_path: Option, - delimiter: String, - storage: Storage, - ) -> Self { - // Create column_map from column names - let column_map = column_names - .iter() - .enumerate() - .map(|(i, name)| (name.clone(), i)) - .collect(); - - // Create full column metadata objects (default to Text type) - let cols = column_names - .iter() - .map(|name| Column { - name: name.clone(), - data_type: DataType::Text, - }) - .collect(); - - Table { - name: name.to_string(), - cols, - column_map, - storage, - file_path, - modified: false, - delimiter, - } - } - /// Create a new table with a schema pub fn new_with_schema( name: &str, @@ -806,10 +940,13 @@ impl Table { name: name.to_string(), cols, column_map, - storage: Storage::new_memory(), + rows: Vec::new(), file_path, modified: true, // Tables created with schema are considered modified delimiter: delimiter.unwrap_or_else(|| ",".to_string()), + format: FileLayout::default(), + comments: CommentLines::default(), + placeholder: false, } } @@ -846,7 +983,7 @@ impl Table { /// of Value enums representing the cell values. This provides read-only /// access to the table data for processing or querying. pub fn rows(&self) -> &[Row] { - self.storage.rows() + &self.rows } /// Get the name of the table @@ -871,7 +1008,7 @@ impl Table { /// Returns the number of rows in the table. This is useful for /// determining the size of the result set or for validation. pub fn row_count(&self) -> usize { - self.storage.row_count() + self.rows.len() } /// Add a row to the table @@ -897,39 +1034,24 @@ impl Table { ))); } - // Ensure storage is mutable (converts mmap to memory if needed) - self.storage.ensure_mutable(); - self.storage.push_row(row); + self.rows.push(row); self.modified = true; Ok(()) } - /// Add a row by cloning values from a slice - /// - /// This method allows reusing a row buffer by cloning values from a slice. - /// It validates that the slice length matches the table column count. - /// - /// # Arguments - /// * `row` - Slice of values to clone and add as a new row - /// - /// # Returns - /// * `Ok(())` if the row was successfully added - /// * `Err` if the slice length doesn't match the table schema - pub fn add_row_from_slice(&mut self, row: &[Value]) -> SqawkResult<()> { - if row.len() != self.column_count() { - return Err(SqawkError::InvalidSqlQuery(format!( - "Row has {} columns, but table '{}' has {} columns", - row.len(), - self.name, - self.column_count() - ))); - } + /// Clear the modified flag (a freshly loaded table is unmodified) + pub fn mark_unmodified(&mut self) { + self.modified = false; + } - // Ensure storage is mutable (converts mmap to memory if needed) - self.storage.ensure_mutable(); - self.storage.push_row(row.to_vec()); - self.modified = true; - Ok(()) + /// Whether this is a `--tabledef` placeholder awaiting its file + pub fn is_placeholder(&self) -> bool { + self.placeholder + } + + /// Mark this table as a `--tabledef` placeholder + pub fn set_placeholder(&mut self, placeholder: bool) { + self.placeholder = placeholder; } /// Get the file path associated with this table @@ -946,15 +1068,6 @@ impl Table { self.file_path.as_ref() } - /// Detach this table from its backing file. - /// - /// Used for tables read from standard input: the data was spooled through - /// a temporary file to be parsed, but that file is not a writeback target, - /// so the table must not claim it as its source. - pub fn detach_file_path(&mut self) { - self.file_path = None; - } - /// Get the delimiter for this table /// /// Returns the delimiter used for this table. @@ -974,6 +1087,23 @@ impl Table { self.delimiter = delimiter; } + /// Layout of the source file (header, line endings, comments) + pub fn file_layout(&self) -> FileLayout { + self.format + } + + /// Record the layout of the source file, for writeback + pub fn set_file_layout(&mut self, format: FileLayout) { + self.format = format; + } + + /// Record the source file's comment lines, for writeback. Comments + /// anchored past the last row are moved to the tail. + pub fn set_comment_lines(&mut self, comments: CommentLines) { + self.comments = comments; + self.comments.flush_from(self.rows.len()); + } + /// Get the index of a column by name /// /// Looks up a column by name and returns its index in the table. @@ -990,39 +1120,75 @@ impl Table { self.column_map.get(name).copied() } - /// Print the table to stdout + /// Print the table to stdout as a delimited file /// - /// Formats and prints the table contents to standard output in comma-delimited format. - /// This is used for displaying query results to the user. + /// Header line, then one record per row, using this table's delimiter + /// with RFC 4180 quoting where a field needs it. NULL prints as an empty + /// field. Returns the raw io::Error so the caller can recognise a closed + /// pipe (`sqawk ... | head`) and stop quietly. + pub fn print_to_stdout(&self) -> std::io::Result<()> { + let stdout = std::io::stdout(); + let mut out = std::io::BufWriter::new(stdout.lock()); + self.write_delimited(&mut out, true, false, false, None)?; + std::io::Write::flush(&mut out) + } + + /// Write the table as a delimited file to `out`. /// - /// # Returns - /// * `Ok(())` if the table was successfully printed - /// * `Err` if there was an error writing to stdout - pub fn print_to_stdout(&self) -> Result<()> { - let delim = &self.delimiter; - - // Print header - let column_names = self.columns(); - for (i, col) in column_names.iter().enumerate() { - if i > 0 { - print!("{}", delim); - } - print!("{}", col); + /// * `header` - emit the column names first + /// * `crlf` - end records with CRLF instead of LF + /// * `comments` - the file will be read with `#` comment skipping, so a + /// leading `#` in a record's first field must be quoted + /// * `lines` - comment lines to put back around the records + pub fn write_delimited( + &self, + out: &mut W, + header: bool, + crlf: bool, + comments: bool, + lines: Option<&CommentLines>, + ) -> std::io::Result<()> { + let delim = delimiter_byte(&self.delimiter); + let w = RecordWriter { + delimiter: delim, + terminator: if crlf { b"\r\n" } else { b"\n" }, + comments, + }; + let empty = CommentLines::default(); + let lines = lines.unwrap_or(&empty); + out.write_all(&lines.head)?; + if header { + let names = self.columns(); + w.write(out, names.iter().map(String::as_str))?; } - println!(); - - // Print rows - for row in self.rows() { - for (i, val) in row.iter().enumerate() { - if i > 0 { - print!("{}", delim); - } - print!("{}", val); + let mut fields: Vec> = Vec::with_capacity(self.cols.len()); + for (i, row) in self.rows.iter().enumerate() { + if let Some(before) = lines.before.get(&i) { + out.write_all(before)?; } - println!(); + fields.clear(); + fields.extend(row.iter().map(Value::field_str)); + w.write(out, fields.iter().map(|f| f.as_ref()))?; + } + // A comment line anchored past the last row (none should be) is + // still written rather than lost. + for (_, before) in lines.before.range(self.rows.len()..) { + out.write_all(before)?; } + out.write_all(&lines.tail) + } - Ok(()) + /// Write the table back to its file format: `write_delimited` with the + /// recorded layout and comment lines. + pub fn write_source_layout(&self, out: &mut W) -> std::io::Result<()> { + let layout = self.file_layout(); + self.write_delimited( + out, + layout.has_header, + layout.crlf, + layout.comments, + Some(&self.comments), + ) } /// Replace all rows with a new set @@ -1033,9 +1199,37 @@ impl Table { /// # Arguments /// * `new_rows` - The new set of rows to replace the existing ones pub fn replace_rows(&mut self, new_rows: Vec) { - // Ensure storage is mutable (converts mmap to memory if needed) - self.storage.ensure_mutable(); - self.storage.replace_rows(new_rows); + self.rows = new_rows; + self.comments.flush_from(self.rows.len()); + self.modified = true; + } + + /// Keep only the rows whose index satisfies `keep` (DELETE). + /// + /// Comment lines anchored to a removed row stay where they were: they + /// move to the next surviving row, or to the tail. + pub fn retain_rows(&mut self, keep: impl Fn(usize) -> bool) { + let old = std::mem::take(&mut self.rows); + let mut before = std::mem::take(&mut self.comments.before); + let mut pending: Vec = Vec::new(); + for (i, row) in old.into_iter().enumerate() { + if let Some(mut lines) = before.remove(&i) { + pending.append(&mut lines); + } + if keep(i) { + if !pending.is_empty() { + self.comments + .before + .insert(self.rows.len(), std::mem::take(&mut pending)); + } + self.rows.push(row); + } + } + for (_, mut lines) in before { + pending.append(&mut lines); + } + pending.append(&mut self.comments.tail); + self.comments.tail = pending; self.modified = true; } @@ -1044,10 +1238,7 @@ impl Table { /// Used by UPDATE. Expressing an update as delete-plus-insert appends the /// new row, which reorders the table -- and with --write, the user's file. pub fn replace_row(&mut self, row_index: usize, row: Row) -> SqawkResult<()> { - self.storage.ensure_mutable(); - let rows = self.storage.rows_mut().ok_or_else(|| { - SqawkError::InvalidSqlQuery("Table storage is not mutable".to_string()) - })?; + let rows = &mut self.rows; if row_index >= rows.len() { return Err(SqawkError::InvalidSqlQuery(format!( "Row index {} out of range for table '{}'", @@ -1075,8 +1266,6 @@ impl Table { self.cols.push(column); let new_index = self.cols.len() - 1; self.column_map.insert(name, new_index); - // Ensure storage is mutable (converts mmap to memory if needed) - self.storage.ensure_mutable(); self.modified = true; } @@ -1095,14 +1284,9 @@ impl Table { let new_index = self.cols.len() - 1; self.column_map.insert(name, new_index); - // Ensure storage is mutable (converts mmap to memory if needed) - self.storage.ensure_mutable(); - // Add default value to all existing rows - if let Some(rows) = self.storage.rows_mut() { - for row in rows { - row.push(default_value.clone()); - } + for row in &mut self.rows { + row.push(default_value.clone()); } self.modified = true; @@ -1111,9 +1295,8 @@ impl Table { /// Clear all rows from the table (TRUNCATE TABLE) pub fn clear_rows(&mut self) -> crate::error::SqawkResult<()> { - // Ensure storage is mutable (converts mmap to memory if needed) - self.storage.ensure_mutable(); - self.storage.replace_rows(Vec::new()); + self.rows.clear(); + self.comments.flush_from(0); self.modified = true; Ok(()) } @@ -1128,7 +1311,89 @@ impl Table { pub fn rows_as_strings(&self) -> Vec> { self.rows() .iter() - .map(|row| row.iter().map(|value| value.to_string()).collect()) + .map(|row| { + row.iter() + .map(|value| value.field_str().into_owned()) + .collect() + }) .collect() } } + +/// The delimiter byte for a table's delimiter string. +/// +/// The CLI validates `-F` to a single ASCII byte (or the two-character escape +/// `\t`), so anything else can only come from a bug; it falls back to comma. +pub fn delimiter_byte(delimiter: &str) -> u8 { + match delimiter.as_bytes() { + [b] => *b, + b"\\t" => b'\t', + _ => b',', + } +} + +/// Writes delimited records with RFC 4180 quoting. +/// +/// A field is quoted when it contains the delimiter, a quote, CR or LF, or +/// has leading or trailing whitespace (which a reader that trims would +/// lose); embedded quotes are doubled. A record consisting of one empty field +/// is written as `""`, since a blank line is skipped on read. The csv crate's +/// writer cannot express the whitespace rule, hence this small writer. +pub struct RecordWriter { + /// Field separator byte + pub delimiter: u8, + /// Record terminator (`\n` or `\r\n`) + pub terminator: &'static [u8], + /// Quote a first field that starts with `#` (see `FileLayout::comments`) + pub comments: bool, +} + +impl RecordWriter { + fn needs_quotes(&self, field: &str, first: bool) -> bool { + let b = field.as_bytes(); + if b.is_empty() { + return false; + } + (first && self.comments && b[0] == b'#') + || b.iter() + .any(|&c| c == self.delimiter || c == b'"' || c == b'\n' || c == b'\r') + || b[0] == b' ' + || b[0] == b'\t' + || b[b.len() - 1] == b' ' + || b[b.len() - 1] == b'\t' + } + + /// Write one record. + pub fn write<'a, W: std::io::Write>( + &self, + out: &mut W, + fields: impl Iterator, + ) -> std::io::Result<()> { + let mut n = 0; + let mut lone_empty = false; + for (i, field) in fields.enumerate() { + if i > 0 { + out.write_all(&[self.delimiter])?; + } + if self.needs_quotes(field, i == 0) { + out.write_all(b"\"")?; + let mut rest = field; + while let Some(pos) = rest.find('"') { + out.write_all(&rest.as_bytes()[..=pos])?; + out.write_all(b"\"")?; + rest = &rest[pos + 1..]; + } + out.write_all(rest.as_bytes())?; + out.write_all(b"\"")?; + } else { + out.write_all(field.as_bytes())?; + } + lone_empty = i == 0 && field.is_empty(); + n = i + 1; + } + if n == 1 && lone_empty { + out.write_all(b"\"\"")?; + } + out.write_all(self.terminator) + } +} diff --git a/src/vm/agg.rs b/src/vm/agg.rs new file mode 100644 index 0000000..e58f46a --- /dev/null +++ b/src/vm/agg.rs @@ -0,0 +1,176 @@ +//! Aggregate accumulators: the running state of one aggregate over one +//! group (AggStep / AggFinal) or one window frame (the window pass). + +use std::collections::HashSet; + +use super::arith::text_of; +use super::bytecode::{ + Register, AGG_AVG, AGG_COUNT, AGG_DISTINCT, AGG_GROUP_CONCAT, AGG_MAX, AGG_MIN, AGG_SUM, + AGG_TOTAL, AGG_TYPE_MASK, +}; +use super::engine::{compare_values, row_key, RowKey}; +use crate::error::{SqawkError, SqawkResult}; +use crate::table::Value; + +/// A running sum: exact while every addend is an integer and the total fits. +#[derive(Debug, Clone, Copy)] +enum Sum { + Int(i64), + Real(f64), +} + +impl Sum { + fn as_f64(self) -> f64 { + match self { + Sum::Int(i) => i as f64, + Sum::Real(f) => f, + } + } +} + +/// An aggregate's running state. +#[derive(Debug, Clone)] +pub(super) struct AggAccumulator { + /// Function type (`AGG_*`, without flags). + func_type: i64, + /// Count of values accumulated (COUNT, and AVG's divisor). + count: i64, + /// SUM / AVG / TOTAL. + sum: Option, + /// MIN / MAX. + best: Option, + /// GROUP_CONCAT. + text: Option, + /// Values already accumulated, for DISTINCT aggregates, keyed as SELECT + /// DISTINCT keys them (1 and 1.0 are one value; text '1' another). `None` + /// when the aggregate is not DISTINCT, so the ordinary path allocates + /// nothing. + seen: Option>, +} + +/// SUM / AVG operand: numbers as they are, numeric text as its number. +/// `None` (skipped) for NULL and for text with no numeric reading. +fn sum_operand(v: &Value) -> Option { + match v.as_numeric()? { + Value::Integer(i) => Some(Sum::Int(i)), + Value::Float(f) => Some(Sum::Real(f)), + _ => None, + } +} + +impl AggAccumulator { + /// A fresh accumulator for function-type word `func_type` (which may + /// carry flags). + pub(super) fn new(func_type: i64) -> Self { + AggAccumulator { + func_type: func_type & AGG_TYPE_MASK, + count: 0, + sum: None, + best: None, + text: None, + seen: (func_type & AGG_DISTINCT != 0).then(Default::default), + } + } + + /// Accumulate one value; `None` is COUNT(*)'s "a row". `sep` is + /// GROUP_CONCAT's separator for this row, if it has one. + pub(super) fn step(&mut self, value: Option, sep: Option) -> SqawkResult<()> { + // DISTINCT: ignore a value already accumulated. NULLs are not + // tracked -- every aggregate already skips them. + if let (Some(seen), Some(val)) = (self.seen.as_mut(), value.as_ref()) { + if !matches!(val, Value::Null) && !seen.insert(row_key(val)) { + return Ok(()); + } + } + match self.func_type { + AGG_COUNT => { + if !matches!(value, Some(Value::Null)) { + self.count += 1; + } + } + AGG_SUM | AGG_AVG | AGG_TOTAL => { + let Some(v) = value.as_ref().and_then(sum_operand) else { + return Ok(()); + }; + self.count += 1; + self.sum = Some(match (self.sum, v) { + (None, v) => v, + (Some(Sum::Int(a)), Sum::Int(b)) => match a.checked_add(b) { + Some(s) => Sum::Int(s), + // SQLite: SUM of integers that overflows is an + // error; AVG and TOTAL carry on in floating point. + None if self.func_type == AGG_SUM => { + return Err(SqawkError::InvalidSqlQuery("integer overflow".into())) + } + None => Sum::Real(a as f64 + b as f64), + }, + (Some(a), b) => Sum::Real(a.as_f64() + b.as_f64()), + }); + } + AGG_MIN | AGG_MAX => { + if let Some(val) = value.filter(|v| !matches!(v, Value::Null)) { + self.count += 1; + let want = if self.func_type == AGG_MIN { + std::cmp::Ordering::Less + } else { + std::cmp::Ordering::Greater + }; + let replace = match &self.best { + None => true, + Some(existing) => compare_values(&val, existing) == want, + }; + if replace { + self.best = Some(val); + } + } + } + AGG_GROUP_CONCAT => { + let Some(val) = value.filter(|v| !matches!(v, Value::Null)) else { + return Ok(()); + }; + let piece = text_of(&Register::from(val)); + match &mut self.text { + None => self.text = Some(piece), + Some(t) => { + match sep { + None => t.push(','), + Some(Value::Null) => {} + Some(s) => t.push_str(&text_of(&Register::from(s))), + } + t.push_str(&piece); + } + } + } + other => { + return Err(SqawkError::VmError(format!( + "Unknown aggregate function type: {}", + other + ))) + } + } + Ok(()) + } + + /// The aggregate's value. Of no rows, COUNT is 0, TOTAL 0.0 and the + /// rest NULL. + pub(super) fn finalize(&self) -> Value { + match self.func_type { + AGG_COUNT => Value::Integer(self.count), + AGG_AVG => match self.sum { + Some(s) if self.count > 0 => Value::Float(s.as_f64() / self.count as f64), + _ => Value::Null, + }, + AGG_TOTAL => Value::Float(self.sum.map_or(0.0, Sum::as_f64)), + AGG_SUM => match self.sum { + Some(Sum::Int(i)) => Value::Integer(i), + Some(Sum::Real(f)) => Value::Float(f), + None => Value::Null, + }, + AGG_GROUP_CONCAT => match &self.text { + Some(t) => Value::String(t.clone().into()), + None => Value::Null, + }, + _ => self.best.clone().unwrap_or(Value::Null), + } + } +} diff --git a/src/vm/arith.rs b/src/vm/arith.rs new file mode 100644 index 0000000..971649e --- /dev/null +++ b/src/vm/arith.rs @@ -0,0 +1,396 @@ +//! Numeric semantics, following SQLite: how text becomes a number, how the +//! arithmetic operators combine integers and reals, and what CAST does. +//! +//! Integers stay 64-bit integers through `+ - * / %`; an operation whose +//! exact result does not fit is computed in floating point instead (SQLite's +//! rule), so `9223372036854775807 + 1` is a REAL rather than a wrapped or +//! saturated integer. Integer `/` truncates toward zero, `%` is the integer +//! remainder, and dividing by zero yields NULL. + +use super::bytecode::Register; +use crate::error::{SqawkError, SqawkResult}; + +/// A number: SQLite's INTEGER or REAL. +#[derive(Debug, Clone, Copy, PartialEq)] +pub(crate) enum Num { + Int(i64), + Real(f64), +} + +impl Num { + pub(crate) fn as_f64(self) -> f64 { + match self { + Num::Int(i) => i as f64, + Num::Real(f) => f, + } + } + + pub(crate) fn into_register(self) -> Register { + match self { + Num::Int(i) => Register::Integer(i), + Num::Real(f) => Register::Float(f), + } + } +} + +/// Length in bytes of the longest prefix of `b` (after leading whitespace) +/// that reads as a number, and whether that prefix is a real (has a point +/// or an exponent). `(start, end, is_real)`; `end == start` when there is no +/// number at all. +fn numeric_prefix(b: &[u8]) -> (usize, usize, bool) { + let mut i = 0; + while i < b.len() && b[i].is_ascii_whitespace() { + i += 1; + } + let start = i; + if i < b.len() && (b[i] == b'+' || b[i] == b'-') { + i += 1; + } + let int_start = i; + while i < b.len() && b[i].is_ascii_digit() { + i += 1; + } + let mut digits = i - int_start; + let mut real = false; + if i < b.len() && b[i] == b'.' { + let frac_start = i + 1; + let mut j = frac_start; + while j < b.len() && b[j].is_ascii_digit() { + j += 1; + } + if digits > 0 || j > frac_start { + digits += j - frac_start; + real = true; + i = j; + } + } + if digits == 0 { + return (start, start, false); + } + // An exponent counts only when at least one digit follows it. + if i < b.len() && (b[i] == b'e' || b[i] == b'E') { + let mut j = i + 1; + if j < b.len() && (b[j] == b'+' || b[j] == b'-') { + j += 1; + } + let exp_start = j; + while j < b.len() && b[j].is_ascii_digit() { + j += 1; + } + if j > exp_start { + real = true; + i = j; + } + } + (start, i, real) +} + +/// Text as a number, the way SQLite's arithmetic reads it: the longest +/// numeric prefix, so `'12abc'` is 12, `' 5 '` is 5, `'1e3'` is the real +/// 1000 and `'abc'` is 0. An integer too large for i64 becomes a real. +pub(crate) fn text_to_num(s: &str) -> Num { + let b = s.as_bytes(); + let (start, end, real) = numeric_prefix(b); + if start == end { + return Num::Int(0); + } + let text = &s[start..end]; + if !real { + if let Ok(i) = text.parse::() { + return Num::Int(i); + } + } + Num::Real(text.parse::().unwrap_or(0.0)) +} + +/// Text as an integer, the way `CAST(text AS INTEGER)` reads it: an optional +/// sign and the digits after it, ignoring anything from the first non-digit +/// on (`'3.9'` is 3, `'1e3'` is 1). Out-of-range values saturate. +pub(crate) fn text_to_int(s: &str) -> i64 { + let b = s.as_bytes(); + let mut i = 0; + while i < b.len() && b[i].is_ascii_whitespace() { + i += 1; + } + let neg = i < b.len() && b[i] == b'-'; + if i < b.len() && (b[i] == b'+' || b[i] == b'-') { + i += 1; + } + let mut acc: i64 = 0; + while i < b.len() && b[i].is_ascii_digit() { + let d = (b[i] - b'0') as i64; + acc = match acc.checked_mul(10).and_then(|v| { + if neg { + v.checked_sub(d) + } else { + v.checked_add(d) + } + }) { + Some(v) => v, + None => return if neg { i64::MIN } else { i64::MAX }, + }; + i += 1; + } + acc +} + +/// 2^63, exactly representable as an f64. +const TWO_63: f64 = 9_223_372_036_854_775_808.0; + +/// A real truncated toward zero, saturating at the ends of the i64 range +/// (SQLite's `CAST(real AS INTEGER)`). +pub(crate) fn real_to_int(f: f64) -> i64 { + if f.is_nan() { + 0 + } else if f >= TWO_63 { + i64::MAX + } else if f <= -TWO_63 { + i64::MIN + } else { + f as i64 + } +} + +/// A register as a number for arithmetic; `None` for NULL. +pub(crate) fn to_num(r: &Register) -> Option { + match r { + Register::Integer(i) => Some(Num::Int(*i)), + Register::Float(f) => Some(Num::Real(*f)), + Register::Boolean(b) => Some(Num::Int(*b as i64)), + Register::String(s) => Some(text_to_num(s)), + Register::Null => None, + } +} + +/// The truth value of a register in a condition, as in SQLite: a number is +/// true when non-zero, and text is read as a number first, so `'x'` is false +/// and `'1x'` true. NULL is not true. +pub(crate) fn is_true(r: &Register) -> bool { + match r { + Register::Integer(v) => *v != 0, + Register::Float(v) => *v != 0.0, + Register::Boolean(v) => *v, + Register::String(s) => text_to_num(s).as_f64() != 0.0, + Register::Null => false, + } +} + +/// An arithmetic operator. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) enum ArithOp { + Add, + Sub, + Mul, + Div, + Rem, +} + +/// `a op b` with SQLite semantics; NULL if either side is NULL, or on +/// division by zero. +pub(crate) fn arith(op: ArithOp, a: &Register, b: &Register) -> Register { + let (Some(x), Some(y)) = (to_num(a), to_num(b)) else { + return Register::Null; + }; + match (x, y) { + (Num::Int(x), Num::Int(y)) => { + let exact = match op { + ArithOp::Add => x.checked_add(y), + ArithOp::Sub => x.checked_sub(y), + ArithOp::Mul => x.checked_mul(y), + ArithOp::Div => { + if y == 0 { + return Register::Null; + } + x.checked_div(y) + } + ArithOp::Rem => { + if y == 0 { + return Register::Null; + } + // i64::MIN % -1 is mathematically 0. + Some(x.checked_rem(y).unwrap_or(0)) + } + }; + match exact { + Some(v) => Register::Integer(v), + None => real_arith(op, x as f64, y as f64), + } + } + (x, y) => real_arith(op, x.as_f64(), y.as_f64()), + } +} + +fn real_arith(op: ArithOp, x: f64, y: f64) -> Register { + let r = match op { + ArithOp::Add => x + y, + ArithOp::Sub => x - y, + ArithOp::Mul => x * y, + ArithOp::Div => { + if y == 0.0 { + return Register::Null; + } + x / y + } + // SQLite takes the remainder of the operands truncated to integers, + // but returns it as a real: 5.5 % 2 is 1.0. + ArithOp::Rem => { + let (ix, iy) = (real_to_int(x), real_to_int(y)); + if iy == 0 { + return Register::Null; + } + let iy = if iy == -1 { 1 } else { iy }; + return Register::Float((ix % iy) as f64); + } + }; + if r.is_nan() { + Register::Null + } else { + Register::Float(r) + } +} + +/// `CAST(value AS type)`. `type` is the compiler's normalized name. +pub(crate) fn cast(r: Register, target: &str) -> Register { + if matches!(r, Register::Null) { + return Register::Null; + } + match target { + "INTEGER" => Register::Integer(match r { + Register::Integer(i) => i, + Register::Float(f) => real_to_int(f), + Register::Boolean(b) => b as i64, + Register::String(s) => text_to_int(&s), + Register::Null => unreachable!(), + }), + "REAL" => Register::Float(match r { + Register::String(s) => text_to_num(&s).as_f64(), + other => to_num(&other).map_or(0.0, Num::as_f64), + }), + // NUMERIC: text that is a number in full becomes that number (a real + // with no fractional part becomes an integer); other text its numeric + // prefix. BOOLEAN has NUMERIC affinity in SQLite, so it yields 1/0 + // for true/false values. + "NUMERIC" | "BOOLEAN" => match r { + Register::String(s) => match text_to_num(&s) { + Num::Real(f) if f.fract() == 0.0 && f.abs() < 9.2e18 => Register::Integer(f as i64), + n => n.into_register(), + }, + Register::Boolean(b) => Register::Integer(b as i64), + other => other, + }, + _ => Register::String(text_of(&r)), + } +} + +/// A non-NULL register's text form, as functions and `||` see it. +pub(crate) fn text_of(r: &Register) -> String { + match r { + Register::String(s) => s.clone(), + Register::Integer(i) => i.to_string(), + Register::Float(f) => f.to_string(), + Register::Boolean(b) => (*b as i64).to_string(), + Register::Null => String::new(), + } +} + +/// Exact comparison of an integer with a real (SQLite's +/// `sqlite3IntFloatCompare`): no rounding of the integer to f64, so +/// i64::MAX is less than the real 2^63. +pub(crate) fn cmp_int_real(i: i64, f: f64) -> std::cmp::Ordering { + use std::cmp::Ordering; + if f.is_nan() { + return Ordering::Greater; + } + if f >= TWO_63 { + return Ordering::Less; + } + if f < -TWO_63 { + return Ordering::Greater; + } + let t = f.trunc(); + match i.cmp(&(t as i64)) { + Ordering::Equal => 0.0f64.partial_cmp(&(f - t)).unwrap_or(Ordering::Equal), + o => o, + } +} + +/// Exact ordering of two numbers. +pub(crate) fn cmp_num(a: Num, b: Num) -> std::cmp::Ordering { + match (a, b) { + (Num::Int(x), Num::Int(y)) => x.cmp(&y), + (Num::Int(x), Num::Real(y)) => cmp_int_real(x, y), + (Num::Real(x), Num::Int(y)) => cmp_int_real(y, x).reverse(), + (Num::Real(x), Num::Real(y)) => x.partial_cmp(&y).unwrap_or(std::cmp::Ordering::Equal), + } +} + +/// The "integer overflow" error SQLite raises for SUM and ABS. +pub(crate) fn overflow() -> SqawkError { + SqawkError::InvalidSqlQuery("integer overflow".into()) +} + +/// `checked` or the integer-overflow error. +pub(crate) fn checked(v: Option) -> SqawkResult { + v.ok_or_else(overflow) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn n(s: &str) -> Num { + text_to_num(s) + } + + #[test] + fn text_prefixes() { + assert_eq!(n("12abc"), Num::Int(12)); + assert_eq!(n(" 5 "), Num::Int(5)); + assert_eq!(n("abc"), Num::Int(0)); + assert_eq!(n("1e3"), Num::Real(1000.0)); + assert_eq!(n("1e"), Num::Int(1)); + assert_eq!(n("007"), Num::Int(7)); + assert_eq!(n(".5"), Num::Real(0.5)); + assert_eq!(n("-"), Num::Int(0)); + assert_eq!(n("0x10"), Num::Int(0)); + assert_eq!(n("3.9x"), Num::Real(3.9)); + assert_eq!(n("99999999999999999999"), Num::Real(1e20)); + } + + #[test] + fn cast_int_prefix() { + assert_eq!(text_to_int("3.9"), 3); + assert_eq!(text_to_int("12abc"), 12); + assert_eq!(text_to_int(" 5 "), 5); + assert_eq!(text_to_int("1e3"), 1); + assert_eq!(text_to_int("-3.9"), -3); + assert_eq!(text_to_int("9999999999999999999"), i64::MAX); + assert_eq!(text_to_int("-9999999999999999999"), i64::MIN); + assert_eq!(real_to_int(1e30), i64::MAX); + assert_eq!(real_to_int(-1e30), i64::MIN); + } + + #[test] + fn integer_ops() { + let i = Register::Integer; + let r = |op, a, b| arith(op, &i(a), &i(b)); + assert!(matches!(r(ArithOp::Div, 5, 2), Register::Integer(2))); + assert!(matches!(r(ArithOp::Div, -7, 2), Register::Integer(-3))); + assert!(matches!(r(ArithOp::Rem, -7, 3), Register::Integer(-1))); + assert!(matches!(r(ArithOp::Div, 5, 0), Register::Null)); + assert!(matches!(r(ArithOp::Add, i64::MAX, 1), Register::Float(_))); + assert!(matches!(r(ArithOp::Div, i64::MIN, -1), Register::Float(_))); + assert!(matches!( + r(ArithOp::Rem, i64::MIN, -1), + Register::Integer(0) + )); + assert!(matches!( + arith(ArithOp::Rem, &Register::Float(5.5), &i(2)), + Register::Float(f) if f == 1.0 + )); + assert!(matches!( + arith(ArithOp::Rem, &i(5), &Register::Float(0.5)), + Register::Null + )); + } +} diff --git a/src/vm/bytecode.rs b/src/vm/bytecode.rs index e7f9dbe..bb5d03f 100644 --- a/src/vm/bytecode.rs +++ b/src/vm/bytecode.rs @@ -23,6 +23,10 @@ pub const AGG_AVG: i64 = 2; pub const AGG_MIN: i64 = 3; /// MAX aggregate function type pub const AGG_MAX: i64 = 4; +/// TOTAL aggregate function type (SUM as a real, 0.0 over no rows) +pub const AGG_TOTAL: i64 = 5; +/// GROUP_CONCAT / STRING_AGG aggregate function type +pub const AGG_GROUP_CONCAT: i64 = 6; /// Flag bit OR-ed into an AggStep/AggFinal function type to request DISTINCT. /// @@ -32,6 +36,10 @@ pub const AGG_MAX: i64 = 4; /// the base function. pub const AGG_DISTINCT: i64 = 0x100; +/// Flag bit: the aggregate has a second argument (GROUP_CONCAT's +/// separator), in the register after the value register. +pub const AGG_SEP: i64 = 0x200; + /// Mask selecting the base aggregate function out of a function-type word. pub const AGG_TYPE_MASK: i64 = 0xFF; @@ -125,9 +133,8 @@ pub enum OpCode { Gt, // Greater than comparison (P1 > P2, result in P3) Ge, // Greater than or equal comparison (P1 >= P2, result in P3) - // Pattern matching operations - Like, // LIKE pattern match (P1 LIKE P4 pattern, result in P3). Case-sensitive. - Glob, // GLOB pattern match (P1 GLOB P4 pattern, result in P3). Unix-style wildcards. + // Pattern matching + Like, // r[P3] = r[P1] LIKE r[P1+1] [ESCAPE r[P1+2]]. P2: bit 0 NOT, bit 1 ILIKE, bit 2 ESCAPE. // Type conversion Cast, // Cast value in P1 to type in P4, result in P2. Types: "INTEGER", "TEXT", "REAL" @@ -143,6 +150,12 @@ pub enum OpCode { // `NOT ` expressible at all. Not, // P2 = NULL if P1 is NULL, else 1 if P1 is falsy, else 0 + // Three-valued AND / OR, modelled on SQLite's OP_And / OP_Or. Both + // operands are evaluated; the result follows the SQL truth tables, so + // NULL AND 0 = 0, NULL AND 1 = NULL, NULL OR 1 = 1, NULL OR 0 = NULL. + And, // P3 = P1 AND P2 + Or, // P3 = P1 OR P2 + // Vector comparison, modelled on SQLite's OP_Compare / OP_Jump. // // Compare sets an internal flag from a pairwise comparison of two register @@ -166,46 +179,67 @@ pub enum OpCode { // Utility opcodes Noop, // No operation - // Set operations - SortResults, // Sort accumulated results. P4 = "col:asc,col:desc,..." over RESULT columns. - Distinct, // Remove duplicate rows from results - Limit, // Limit results to P1 rows. P2 = offset (rows to skip, already applied). Post-processing opcode. - Intersect, // Keep only rows that exist in both left and right result sets - Except, // Keep only rows from left that don't exist in right result set - // Join operations - NullRow, // Load NULL values into registers P1 through P1+P2-1 (P2 = count) - RewindInner, // Rewind cursor P1, used for inner loop of nested join - MarkMatch, // Set register P1 to 1 to indicate a match was found - CheckMatch, // If register P1 is 0 (no match), jump to P2; reset register to 0 - - // Subquery/Coroutine operations (inspired by SQLite) - InitCoroutine, // P1 = register to store return address, P2 = jump over coroutine, P3 = coroutine entry - Yield, // Swap program counter with register P1 (coroutine yield) - EndCoroutine, // Jump back to Yield that invoked this coroutine, set P1 for next Yield return - Once, // First time: continue. Subsequently: jump to P2. Used for one-time init. - - // Subquery result operations - AggStep, // P1 = function type, P2 = value reg, P3 = accumulator reg. Step aggregate. - AggFinal, // P1 = accumulator reg, P2 = result reg. Finalize aggregate. - AggReset, // P1 = accumulator reg. Reset accumulator to initial state. - Exists, // P1 = flag reg (set to 1 if cursor P2 has rows, else 0) - NotExists, // P1 = flag reg (set to 1 if cursor P2 has no rows, else 0) + // + // NullRow follows SQLite's OP_NullRow: it puts cursor P1 on a virtual + // all-NULL row, so every Column read from it yields NULL, and leaves it + // exhausted, so a following Next/HashNext falls through. That is how an + // outer join's unmatched side is NULL-extended without copying columns + // into registers first. + NullRow, // Put cursor P1 on an all-NULL row (Column -> NULL; Next falls through) + + // Hash join, an ephemeral hash index over a cursor's rows. + // + // HashInsert is run once per row of the inner table to build the index; + // HashProbe then positions the cursor on the first row whose key equals + // the probe key and HashNext walks the remaining matches, so the loop + // body reads the inner row with ordinary Column instructions. Keys are + // normalized so that every pair of values the Eq opcode calls equal lands + // in the same bucket (1 and 1.0, 1 and '1'); the join condition is still + // re-checked on each candidate, so a bucket may over-approximate. A NULL + // key is never inserted and never matches. + HashInsert, // Add cursor P1's current row to its hash index under key r[P2..P2+P3] + HashProbe, // Position cursor P1 on the first row matching key r[P3..P3+P4]; jump to P2 if none + HashNext, // Advance cursor P1 to its next probe match and jump to P2; fall through when done + + // Per-row matched bits, for FULL JOIN's pass over unmatched right rows. + MarkRowMatched, // Record that cursor P1's current row matched (no-op on a NULL row) + IfRowMatched, // Jump to P2 if cursor P1's current row was marked matched + + // Subqueries: an uncorrelated subquery's fragment, and a subquery's + // cursor setup, run only the first time they are reached. + Once, // First time: continue. Subsequently: jump to P2. + + // Aggregates + AggStep, // P1 = function type, P2 = value reg, P3 = accumulator reg. Step aggregate. + AggFinal, // P1 = accumulator reg, P2 = result reg. Finalize aggregate. + AggReset, // P1 = accumulator reg. Reset accumulator to initial state. // Counter operations (LIMIT/OFFSET) DecrJumpZero, // Decrement register P1. If result is zero, jump to P2. + MustBeInt, // Make r[P1] an integer (integral REAL or integer text converts); else error. P4 = clause name. // Sorter operations (ORDER BY) SorterOpen, // Open a sorter. P1 = sorter ID, P2 = number of columns, P4 = sort key spec SorterInsert, // Insert row into sorter P1. P2 = start register, P3 = column count - SorterSort, // Sort the sorter P1. Must be called before SorterNext. + SorterSort, // Sort the sorter P1. Must be called before SorterNext. Jump to P2 (if > 0) when empty. SorterData, // Copy current sorter P1 row to registers starting at P2. P3 = column count. SorterNext, // Advance sorter P1. Jump to P2 if more rows, else continue. // Ephemeral table operations (for ORDER BY results, temp storage) // Ephemeral tables use cursor IDs and work with Column/Next/Rewind opcodes - OpenEphemeral, // Open ephemeral cursor P1 with P2 columns. P4 = sort key spec for ordering. + // + // With P3 = 1 an ephemeral table is KEYED, like SQLite's ephemeral index: + // it holds each distinct row once (IdxInsert of a row already present is a + // no-op), Found/NotFound test membership, IdxDelete removes a row, and + // Rewind walks the rows in sorted order. DISTINCT and the compound + // operators (UNION, INTERSECT, EXCEPT) are built from these. Rows are + // equal when every column is: NULL equals NULL and 1 equals 1.0. + OpenEphemeral, // Open ephemeral cursor P1 with P2 columns. P4 = sort key spec; P3 = 1: keyed. IdxInsert, // Insert into ephemeral cursor P1. P2 = start register, P3 = column count. + IdxDelete, // Remove row r[P2..P2+P3] from keyed ephemeral cursor P1, if present. + Found, // Jump to P2 if row r[P3..P3+P4] is in keyed ephemeral cursor P1. + NotFound, // Jump to P2 if row r[P3..P3+P4] is NOT in keyed ephemeral cursor P1. Sort, // Sort ephemeral cursor P1. Jump to P2 when empty, else continue. Sequence, // P1 = ephemeral cursor, P2 = dest register. Generate next sequence number. @@ -215,14 +249,10 @@ pub enum OpCode { AlterTableAdd, // Add column to table. P4 = "table_name:col_name:col_type" Truncate, // Remove all rows from table. P4 = table name - // String functions - StringFunc, // P1 = src reg, P2 = dest reg, P3 = arg2 reg (for SUBSTR/REPLACE), P4 = function name - - // Math functions - MathFunc, // P1 = src reg, P2 = dest reg, P4 = function name (ABS, ROUND, CEIL, FLOOR) - - // Date/Time functions - DateFunc, // P1 = src reg (optional), P2 = dest reg, P4 = function name (DATE, TIME, NOW) + // Scalar function call, modelled on SQLite's OP_Function: the body is + // looked up by name in the built-in table (`func.rs`) when the program + // is loaded. + Function, // r[P3] = P4(r[P1], ..., r[P1+P2-1]) // Arithmetic operations Add, // P1 = left reg, P2 = right reg, P3 = dest reg. dest = left + right @@ -231,11 +261,143 @@ pub enum OpCode { Divide, // P1 = left reg, P2 = right reg, P3 = dest reg. dest = left / right Remainder, // P1 = left reg, P2 = right reg, P3 = dest reg. dest = left % right - // Window function operations - WindowAggStep, // P1 = func type, P2 = value reg, P3 = accum reg, P4 = window spec. Step window aggregate. - WindowValue, // P1 = accum reg, P2 = dest reg, P3 = func type. Get current window value. - WindowFinalize, // P1 = result column index. Give every row of a partition that partition's - // final window value, which for an unordered frame is the partition total. + // Window functions, modelled on SQLite's window pass over a sorted + // ephemeral table: the rows of the query (every value the output needs, + // plus each window's partition keys, order keys and function arguments) + // are first collected into an ephemeral table; WindowCompute then sorts + // that table by one window definition's partition and order keys, walks + // each partition and writes every function of that definition into its + // result column of each row. The output stage reads the rows back. + WindowCompute, // Compute window definition Program::windows[P2] over ephemeral cursor P1. +} + +/// One sort key: a column, its direction, and where NULLs go. +/// +/// NULL sorts first ascending and last descending unless the ORDER BY term +/// says `NULLS FIRST` / `NULLS LAST` (SQLite's default and syntax). +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct KeyOrder { + pub col: usize, + pub asc: bool, + pub nulls_first: bool, +} + +impl KeyOrder { + /// A key with an optional explicit NULLS FIRST (`Some(true)`) / LAST. + pub fn new(col: usize, asc: bool, nulls_first: Option) -> Self { + KeyOrder { + col, + asc, + nulls_first: nulls_first.unwrap_or(asc), + } + } + + /// This key in a SorterOpen / OpenEphemeral P4 spec: `col:asc|desc`, + /// plus `:nf` / `:nl` when NULL placement is not the default. + pub fn spec(&self) -> String { + let dir = if self.asc { "asc" } else { "desc" }; + match (self.asc, self.nulls_first) { + (true, true) | (false, false) => format!("{}:{}", self.col, dir), + (_, true) => format!("{}:{}:nf", self.col, dir), + (_, false) => format!("{}:{}:nl", self.col, dir), + } + } + + /// Parse a comma-separated spec (see [`KeyOrder::spec`]). + pub fn parse_spec(spec: &str) -> Vec { + spec.split(',') + .filter_map(|k| { + let mut parts = k.split(':'); + let col = parts.next()?.parse::().ok()?; + let asc = parts.next()? != "desc"; + let nulls = match parts.next() { + Some("nf") => Some(true), + Some("nl") => Some(false), + _ => None, + }; + Some(KeyOrder::new(col, asc, nulls)) + }) + .collect() + } +} + +/// A window definition, as WindowCompute executes it: one sort order +/// (PARTITION BY + ORDER BY) and every window function that shares it. +/// +/// All positions are column indices of the ephemeral table the window stage +/// collected its rows into. +#[derive(Debug, Clone)] +pub struct WindowDef { + /// PARTITION BY key columns. + pub partition: Vec, + /// ORDER BY keys. Rows equal on all of them are peers. + pub order: Vec, + pub funcs: Vec, +} + +/// One window function call of a [`WindowDef`]. +#[derive(Debug, Clone)] +pub struct WindowFunc { + pub kind: WindowFuncKind, + /// Argument columns, in call order. + pub args: Vec, + /// The frame; only aggregates and FIRST_VALUE / LAST_VALUE / NTH_VALUE + /// look at it. + pub frame: WindowFrame, + /// Column receiving the function's value for each row. + pub result: usize, +} + +/// What a [`WindowFunc`] computes. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum WindowFuncKind { + RowNumber, + Rank, + DenseRank, + PercentRank, + CumeDist, + Ntile, + Lag, + Lead, + FirstValue, + LastValue, + NthValue, + /// An aggregate (`AGG_*` function type) over the frame. `COUNT(*)` is + /// `AGG_COUNT` with no argument. + Agg(i64), +} + +/// A window frame: the rows, relative to the current row, that an aggregate +/// or a value function sees. Bounds are already validated (start no later +/// in the list UNBOUNDED PRECEDING < n PRECEDING < CURRENT ROW < n FOLLOWING +/// < UNBOUNDED FOLLOWING than end). +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct WindowFrame { + /// ROWS (`true`) or RANGE. A RANGE frame's bounds are CURRENT ROW or + /// UNBOUNDED only, so it is defined by peer groups. + pub rows: bool, + pub start: FrameBound, + pub end: FrameBound, +} + +impl WindowFrame { + /// The default frame: RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW, + /// which without ORDER BY is the whole partition (every row a peer). + pub const DEFAULT: WindowFrame = WindowFrame { + rows: false, + start: FrameBound::UnboundedPreceding, + end: FrameBound::CurrentRow, + }; +} + +/// One end of a [`WindowFrame`]. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum FrameBound { + UnboundedPreceding, + Preceding(u64), + CurrentRow, + Following(u64), + UnboundedFollowing, } /// A SQL VM instruction with opcode and parameters @@ -321,6 +483,8 @@ pub struct Program { /// hold register numbers (`Integer 999999 -> r` being the obvious case), /// so scanning for the maximum both over- and under-estimates. pub register_count: i64, + /// Window definitions, indexed by WindowCompute's P2. + pub windows: Vec, } impl Default for Program { @@ -335,6 +499,7 @@ impl Program { instructions: Vec::with_capacity(DEFAULT_INSTRUCTION_CAPACITY), result_schema: ResultSchema::new(), register_count: 0, + windows: Vec::new(), } } diff --git a/src/vm/compiler.rs b/src/vm/compiler.rs index 14a1001..5073fca 100644 --- a/src/vm/compiler.rs +++ b/src/vm/compiler.rs @@ -5,43 +5,30 @@ //! It implements a visitor pattern to walk the AST generated by sqlparser. use super::ast_compat::{ - create_table_options, delete_from, func_args, func_is_distinct, group_by_exprs, insert_target, - is_group_by_all, is_order_by_all, object_name_last, order_by_is_asc, query_limit, query_offset, - query_order_by, + create_table_options, delete_from, func_args, func_is_distinct, insert_target, object_name_last, }; use sqlparser::ast::{ BinaryOperator, CaseWhen, DataType as SqlDataType, Expr, Function, FunctionArg, - FunctionArgExpr, Ident, JoinConstraint, ObjectName, Query, Select, SelectItem, SetExpr, - SetOperator, SetQuantifier, Statement, TableWithJoins, UnaryOperator, Value, ValueWithSpan, + FunctionArgExpr, FunctionArguments, Ident, ObjectName, SelectItem, + SelectItemQualifiedWildcardKind, Statement, TrimWhereField, UnaryOperator, Value, + ValueWithSpan, }; use sqlparser::dialect::HiveDialect; use sqlparser::parser::Parser; use std::collections::HashMap; -use super::bytecode::{ - Instruction, OpCode, Program, ResultSchema, AGG_AVG, AGG_COUNT, AGG_DISTINCT, AGG_MAX, AGG_MIN, - AGG_SUM, -}; +use super::bytecode::{Instruction, OpCode, Program, AGG_COUNT, AGG_DISTINCT}; +use super::compiler_check::{check_create_table, check_delete, check_insert, check_update}; +use super::compiler_from::{expr_children, FromPlan}; +use super::compiler_subquery::Sub; +use super::compiler_window::window_call_children; +use super::func; use crate::aggregate::AggregateFunction; use crate::capacity::DEFAULT_CURSOR_CAPACITY; use crate::database::Database; use crate::error::{SqawkError, SqawkResult}; use crate::table::{DataType, Table, Value as TableValue}; -/// Result type for multi-table projection resolution -/// Contains: (columns_per_table, output_column_refs, schema, is_wildcard) -pub(crate) type MultiTableProjection = (Vec>, Vec<(usize, usize)>, ResultSchema, bool); -/// Outer column reference found in a correlated subquery (Phase 4B) -#[derive(Debug, Clone)] -struct OuterColumnRef { - /// The table qualifier (alias or table name) the outer query used. - /// - /// Only the qualifier is needed: which column it names is resolved by - /// NameCtx against the outer table, so recording it here would be a second - /// copy of that lookup. - qualifier: String, -} - /// A symbolic jump target. /// /// Jump destinations used to be computed arithmetically, as offsets from @@ -59,17 +46,10 @@ struct OuterColumnRef { pub(crate) struct Label(usize); /// One FROM source visible to expression compilation. +#[derive(Clone)] pub(crate) struct Src<'t> { /// VM cursor id for this source. pub cursor: i64, - /// Base register holding this source's columns, when the loop has already - /// materialized them. - /// - /// Multi-table joins load every column into registers before evaluating - /// join conditions, so a column reference there is a register offset - /// rather than a `Column` read from a cursor. Modelling both here means - /// one expression compiler serves both loop shapes. - pub reg_base: Option, /// The table behind the cursor. pub table: &'t Table, /// How the query refers to it: the alias if there is one, else the table @@ -78,25 +58,40 @@ pub(crate) struct Src<'t> { pub refname: String, } +/// One output column of a SELECT list, after `*` expansion. +pub(crate) struct OutCol { + /// The value, as an expression over the FROM sources. + pub expr: Expr, + /// Result header. + pub name: String, + /// The `AS` alias, which ORDER BY may name. + pub alias: Option, +} + +/// A column merged by `JOIN ... USING (col)` or NATURAL JOIN. +/// +/// SQL gives such a join ONE output column per USING name, so an unqualified +/// reference is not ambiguous even though both tables have the column. Its +/// value is the first non-NULL of the joined columns, which for an inner or +/// LEFT join is simply the left table's value and for RIGHT/FULL picks up the +/// right side's value on NULL-extended rows. +#[derive(Clone, Debug)] +pub(crate) struct UsingCol { + /// Column name, ASCII-lowercased. + pub name: String, + /// The merged columns as (cursor, column index), leftmost first. + pub refs: Vec<(i64, usize)>, +} + /// A resolved column: where its value lives at run time. -#[derive(Clone, Copy, Debug)] +#[derive(Clone, Copy, Debug, PartialEq, Eq)] pub(crate) enum ColumnRef { /// Read column `col` from cursor `cursor`. Cursor { cursor: i64, col: usize }, /// The value is already in this register. Register(i64), -} - -impl ColumnRef { - fn new(src: &Src<'_>, col: usize) -> Self { - match src.reg_base { - Some(base) => ColumnRef::Register(base + col as i64), - None => ColumnRef::Cursor { - cursor: src.cursor, - col, - }, - } - } + /// A USING-merged column: index into [`NameCtx::using`]. + Using(usize), } /// Name resolution scope for [`SqlCompiler::code_expr`]. @@ -106,62 +101,76 @@ impl ColumnRef { /// alias, or a post-aggregation scope. Everything a name can resolve against /// lives here instead, so one compiler serves the SELECT list, WHERE, HAVING, /// ORDER BY and SET clauses. +#[derive(Clone)] pub(crate) struct NameCtx<'t> { /// FROM sources in cursor order. Empty for `SELECT 1` (no FROM). pub srcs: Vec>, - /// Enclosing query's sources, for a correlated subquery. + /// The enclosing query's scope, for a subquery. /// /// SQL resolves a name in the innermost scope first and only then looks - /// outward, so these are consulted only when `srcs` has no match. Keeping - /// them in a separate list rather than appending to `srcs` preserves that - /// precedence, and keeps the ambiguity check meaningful: two tables in one - /// FROM are ambiguous, but an inner name shadowing an outer one is not. - pub outer: Vec>, - - /// Aggregate results available to this expression, keyed by the rendered - /// call text (`"sum(salary)"`). + /// outward, so the enclosing scope is consulted only when this one has no + /// match. Keeping it as a separate link rather than appending its sources + /// preserves that precedence, and keeps the ambiguity check meaningful: + /// two tables in one FROM are ambiguous, but an inner name shadowing an + /// outer one is not. A name that resolves through this link makes the + /// subquery correlated. + pub outer: Option>>, + + /// Number of USING columns in all enclosing scopes. [`ColumnRef::Using`] + /// indexes the chain as one list, outermost scope first, so a merged + /// column of an enclosing join can be referenced from a subquery. + pub using_base: usize, + + /// Values already computed into registers, keyed by the rendered + /// expression text (`"sum(salary)"`, or a GROUP BY key such as + /// `"upper(department)"`). /// /// After aggregation an aggregate is a value like any other, so an /// expression may contain one -- `SUM(salary) + 1`. Binding them here lets /// the ordinary expression compiler handle the surrounding arithmetic /// instead of needing a separate post-aggregation compiler. pub agg_bindings: Vec<(String, i64)>, + + /// Columns merged by USING / NATURAL JOIN. + pub using: Vec, + + /// Post-aggregation column scope. + /// + /// `None` for an ordinary row scope. `Some` once rows have been grouped: + /// only GROUP BY key columns still have a single value per output row, so + /// a column reference must be one of these (bound to the register holding + /// the key) or it is an error -- reading the cursor at that point would + /// return whatever row the scan happened to stop on. + pub col_bindings: Option>, } impl<'t> NameCtx<'t> { /// The common single-table scope, equivalent to the old /// `(table, cursor_idx)` pair. pub(crate) fn single(table: &'t Table, cursor: i64) -> Self { - NameCtx { - srcs: vec![Src { - cursor, - reg_base: None, - table, - refname: table.name().to_ascii_lowercase(), - }], - outer: Vec::new(), - agg_bindings: Vec::new(), - } + Self::multi(vec![Src { + cursor, + table, + refname: table.name().to_ascii_lowercase(), + }]) } /// A scope over several FROM sources, for joins. pub(crate) fn multi(srcs: Vec>) -> Self { NameCtx { srcs, - outer: Vec::new(), + outer: None, + using_base: 0, agg_bindings: Vec::new(), + using: Vec::new(), + col_bindings: None, } } /// A scope with no FROM sources, for constant expressions. - #[allow(dead_code)] // used by constant-expression contexts pub(crate) fn empty() -> Self { - NameCtx { - srcs: Vec::new(), - outer: Vec::new(), - agg_bindings: Vec::new(), - } + Self::multi(Vec::new()) } /// Normalized key for an aggregate call, used to bind and to look up. @@ -169,8 +178,11 @@ impl<'t> NameCtx<'t> { expr.to_string().to_ascii_lowercase() } - /// Register holding the result of an already-computed aggregate. - fn lookup_agg(&self, expr: &Expr) -> Option { + /// Register holding an already-computed value for `expr`, if bound. + pub(crate) fn lookup_agg(&self, expr: &Expr) -> Option { + if self.agg_bindings.is_empty() { + return None; + } let key = Self::agg_key(expr); self.agg_bindings .iter() @@ -178,122 +190,163 @@ impl<'t> NameCtx<'t> { .map(|(_, r)| *r) } - /// Resolve an unqualified column name across every source. - /// - /// Case-insensitive: `SELECT NAME` already worked while - /// `WHERE NAME = 'x'` did not, because the projection path lowercased and - /// `find_column_index` compared exactly. - /// - /// Returns `Err` if no source has the column, or if more than one does -- - /// silently picking the first would make the result depend on FROM order. - fn resolve_unqualified(&self, name: &str) -> SqawkResult { - let want = name.to_ascii_lowercase(); - let mut found: Option = None; - for src in &self.srcs { - for (i, col) in src.table.column_metadata().iter().enumerate() { - if col.name.to_ascii_lowercase() == want { - if found.is_some() { - return Err(SqawkError::InvalidSqlQuery(format!( - "Column '{}' is ambiguous across the tables in FROM", - name - ))); - } - found = Some(ColumnRef::new(src, i)); - break; - } - } + /// Apply the post-aggregation scope to a resolved column. + fn bind(&self, col: ColumnRef, name: &str) -> SqawkResult { + match &self.col_bindings { + None => Ok(col), + Some(bound) => bound + .iter() + .find(|(c, _)| *c == col) + .map(|(_, reg)| ColumnRef::Register(*reg)) + .ok_or_else(|| { + SqawkError::InvalidSqlQuery(format!( + "'{}' must appear in GROUP BY or be used in an aggregate", + name + )) + }), } - if let Some(col) = found { - return Ok(col); + } + + /// Attach the enclosing query's scope. + pub(crate) fn with_outer(mut self, outer: Option>>) -> Self { + self.using_base = outer.as_ref().map_or(0, |o| o.using_base + o.using.len()); + self.outer = outer; + self + } + + /// The USING column with chain-wide index `i`. + pub(crate) fn using_col(&self, i: usize) -> &UsingCol { + if i >= self.using_base { + &self.using[i - self.using_base] + } else { + self.outer + .as_ref() + .expect("USING index below using_base implies an outer scope") + .using_col(i) } - // Not in this scope: look outward, nearest first. - for src in &self.outer { - for (i, col) in src.table.column_metadata().iter().enumerate() { - if col.name.to_ascii_lowercase() == want { - return Ok(ColumnRef::new(src, i)); - } - } + } + + /// The scope `depth` links outward (0 = this one). + pub(crate) fn scope_at(&self, depth: usize) -> &NameCtx<'t> { + let mut s = self; + for _ in 0..depth { + s = s.outer.as_deref().expect("depth within the scope chain"); } - Err(SqawkError::ColumnNotFound(name.to_string())) + s } - /// Resolve a `table.column` (or `alias.column`) reference. - fn resolve_qualified(&self, qualifier: &str, name: &str) -> SqawkResult { - let qual = qualifier.to_ascii_lowercase(); + /// Look a column up in THIS scope only, without the post-aggregation + /// binding. + /// + /// `Ok(None)`: no such column here. `Err`: the name is ambiguous across + /// this scope's tables, or the qualifier names a table here that lacks + /// the column. + fn find_here(&self, qualifier: Option<&str>, name: &str) -> SqawkResult> { let want = name.to_ascii_lowercase(); + let column_of = |src: &Src<'_>| { + src.table + .column_metadata() + .iter() + .position(|c| c.name.to_ascii_lowercase() == want) + }; + if let Some(qualifier) = qualifier { + let qual = qualifier.to_ascii_lowercase(); + let Some(src) = self.srcs.iter().find(|s| s.refname == qual) else { + return Ok(None); + }; + return match column_of(src) { + Some(col) => Ok(Some(ColumnRef::Cursor { + cursor: src.cursor, + col, + })), + None => Err(SqawkError::ColumnNotFound(format!( + "{}.{}", + qualifier, name + ))), + }; + } + // A USING-merged column is one column by definition, never ambiguous. + if let Some(i) = self.using.iter().position(|u| u.name == want) { + return Ok(Some(ColumnRef::Using(self.using_base + i))); + } + let mut found: Option = None; for src in &self.srcs { - if src.refname != qual { - continue; - } - for (i, col) in src.table.column_metadata().iter().enumerate() { - if col.name.to_ascii_lowercase() == want { - return Ok(ColumnRef::new(src, i)); + if let Some(col) = column_of(src) { + if found.is_some() { + // Silently picking the first would make the result + // depend on FROM order. + return Err(SqawkError::InvalidSqlQuery(format!( + "Column '{}' is ambiguous across the tables in FROM", + name + ))); } + found = Some(ColumnRef::Cursor { + cursor: src.cursor, + col, + }); } - return Err(SqawkError::ColumnNotFound(format!( - "{}.{}", - qualifier, name - ))); } - // Qualifier not in this scope: try the enclosing one, which is how a - // correlated subquery refers to its outer row. - for src in &self.outer { - if src.refname != qual { - continue; + Ok(found) + } + + /// Resolve a column reference through the scope chain, innermost scope + /// first, as SQL does. Returns the column -- with the post-aggregation + /// binding of the scope it was found in applied -- and how many scopes + /// outward that was (0: this scope; more: an outer reference, which + /// makes the subquery correlated). + /// + /// Matching is case-insensitive. A qualified name whose qualifier no + /// scope knows falls back, on a single-table scope, to the unqualified + /// name, so `employees.age` still works when the table was given an + /// alias. With several tables a wrong qualifier is an error: the fallback + /// could silently read the same-named column of another table. + pub(crate) fn lookup( + &self, + qualifier: Option<&str>, + name: &str, + ) -> SqawkResult<(ColumnRef, usize)> { + let label = match qualifier { + Some(q) => format!("{}.{}", q, name), + None => name.to_string(), + }; + let mut scope = Some(self); + let mut depth = 0; + while let Some(s) = scope { + if let Some(col) = s.find_here(qualifier, name)? { + return Ok((s.bind(col, &label)?, depth)); } - for (i, col) in src.table.column_metadata().iter().enumerate() { - if col.name.to_ascii_lowercase() == want { - return Ok(ColumnRef::new(src, i)); - } + scope = s.outer.as_deref(); + depth += 1; + } + if qualifier.is_some() && self.srcs.len() <= 1 { + if let Some(col) = self.find_here(None, name)? { + return Ok((self.bind(col, &label)?, 0)); } } - // Unknown qualifier entirely: fall back to an unqualified match so a - // single-table query can still say `employees.age` when the cursor was - // opened under a different refname. - self.resolve_unqualified(name) - } -} - -/// Classify a join operator for the inner-join rewrite. -/// -/// `Some(Some(c))` = inner join with constraint `c`; `Some(None)` = cross join; -/// `None` = not rewritable (outer joins and the exotic forms). -fn classify_join_for_rewrite(op: &sqlparser::ast::JoinOperator) -> Option> { - use sqlparser::ast::JoinOperator as J; - match op { - J::Join(c) | J::Inner(c) => Some(Some(c)), - J::CrossJoin(_) => Some(None), - _ => None, + Err(SqawkError::ColumnNotFound(label)) } -} -/// Validate the argument count of a math function (ABS, ROUND, CEIL, -/// CEILING, FLOOR). -/// -/// Shared because expressions reach the `MathFunc` opcode through two entry -/// points -- `code_expr`'s `compile_func_math` and the older `compile_expr`, -/// which handles a SELECT list with no FROM clause. When only the first -/// validated arity, `SELECT ABS(-5, 2)` quietly returned 5 while -/// `SELECT ABS(-5, 2) FROM t` errored. -fn check_math_func_arity(func_name: &str, arg_count: usize) -> SqawkResult<()> { - if arg_count == 0 { - return Err(SqawkError::InvalidSqlQuery(format!( - "{} requires one argument", - func_name - ))); + /// Resolve an unqualified column name in this scope only (no outward + /// lookup, no post-aggregation binding). + pub(crate) fn resolve_unqualified(&self, name: &str) -> SqawkResult { + self.find_here(None, name)? + .ok_or_else(|| SqawkError::ColumnNotFound(name.to_string())) } - // ROUND is the only one of these taking an optional precision argument. - let max_args = if func_name == "ROUND" { 2 } else { 1 }; - if arg_count > max_args { - return Err(SqawkError::InvalidSqlQuery(if max_args == 1 { - format!("{} takes exactly one argument", func_name) - } else { - format!("{} takes at most two arguments", func_name) - })); + /// Resolve a `table.column` (or `alias.column`) reference in this scope + /// only, with the single-table fallback of [`Self::lookup`]. + pub(crate) fn resolve_qualified(&self, qualifier: &str, name: &str) -> SqawkResult { + if let Some(col) = self.find_here(Some(qualifier), name)? { + return Ok(col); + } + if self.srcs.len() <= 1 { + return self.resolve_unqualified(name); + } + Err(SqawkError::ColumnNotFound(format!( + "{}.{}", + qualifier, name + ))) } - - Ok(()) } /// Reject a scale or TO-datetime argument on CEIL/FLOOR. @@ -334,8 +387,15 @@ pub struct SqlCompiler<'a> { /// Whether to generate verbose bytecode with detailed comments pub(crate) verbose: bool, - /// Current outer table alias for correlated subquery detection - pub(crate) current_outer_alias: Option, + /// While a subquery is being compiled: the scope of the expression that + /// contains it, which becomes the outer link of the subquery's FROM + /// scope. `None` at statement level. + pub(crate) outer_scope: Option>, + + /// One flag per subquery being compiled, innermost last: set when the + /// subquery (or one nested in it) references a column of an enclosing + /// query, i.e. when it is correlated. + pub(crate) correlated: Vec, /// Resolved address of each label, indexed by label id. `None` until /// [`SqlCompiler::resolve`] binds it. @@ -347,8 +407,33 @@ pub struct SqlCompiler<'a> { /// Three-way `Jump`s awaiting resolution: /// (instruction address, less, equal, greater). pending_jumps_three: Vec<(usize, Label, Label, Label)>, + + /// Next free ephemeral cursor id. Starts above any FROM source's cursor + /// (sources are numbered from 0), so a temporary table never collides + /// with a scan. + next_cursor: i64, + + /// Next free sorter id. + next_sorter: i64, + + /// While an INSERT ... SELECT or CREATE TABLE ... AS is compiled: where + /// `Sink::Table` sends the query's rows. + pub(crate) insert_target: Option, +} + +/// The table an INSERT ... SELECT or CREATE TABLE ... AS writes into. +#[derive(Debug, Clone)] +pub(crate) struct InsertTarget { + /// Cursor opened for writing on the table. + pub cursor: i64, + /// The table's column count, and the column each query column fills + /// (the others are NULL). `None`: the query's columns, as they are. + pub columns: Option<(usize, Vec)>, } +/// First cursor id handed out by [`SqlCompiler::alloc_cursor`]. +const FIRST_TEMP_CURSOR: i64 = 1000; + impl<'a> SqlCompiler<'a> { /// Create a new SQL compiler pub fn new(database: &'a Database, verbose: bool) -> Self { @@ -358,13 +443,29 @@ impl<'a> SqlCompiler<'a> { program: Program::new(), register_counter: 0, verbose, - current_outer_alias: None, + outer_scope: None, + correlated: Vec::new(), label_addrs: Vec::new(), pending_jumps: Vec::new(), pending_jumps_three: Vec::new(), + next_cursor: FIRST_TEMP_CURSOR, + next_sorter: 0, + insert_target: None, } } + /// A fresh cursor id for an ephemeral table. + pub(crate) fn alloc_cursor(&mut self) -> i64 { + self.next_cursor += 1; + self.next_cursor - 1 + } + + /// A fresh sorter id. + pub(crate) fn alloc_sorter(&mut self) -> i64 { + self.next_sorter += 1; + self.next_sorter - 1 + } + /// Create a new, unresolved jump target. pub(crate) fn label(&mut self) -> Label { self.label_addrs.push(None); @@ -499,26 +600,6 @@ impl<'a> SqlCompiler<'a> { )); } - /// Helper to emit Column opcodes for multiple columns - pub(crate) fn emit_column_loads( - &mut self, - cursor: i64, - cols: &[usize], - start_reg: i64, - label: &str, - ) { - for (i, col_idx) in cols.iter().enumerate() { - self.emit( - OpCode::Column, - cursor, - *col_idx as i64, - start_reg + i as i64, - None, - &format!("r[{}] = {}.col[{}]", start_reg + i as i64, label, col_idx), - ); - } - } - /// The AggStep/AggFinal function-type word for an aggregate CALL, /// including its DISTINCT flag. /// @@ -528,8 +609,7 @@ impl<'a> SqlCompiler<'a> { /// a grouped query -- and HAVING, which did set the flag, then failed to /// match the aggregate it had been given. pub(crate) fn agg_func_type_for(func: &Function) -> i64 { - let name = func.name.to_string().to_uppercase(); - Self::agg_func_type(&name) + Self::agg_func_type(&func.name.to_string()) | if func_is_distinct(func) { AGG_DISTINCT } else { @@ -537,16 +617,13 @@ impl<'a> SqlCompiler<'a> { } } - /// Convert aggregate function name to type code + /// Convert an aggregate function name (any case) to its type code. + /// + /// Callers have already established that `name` is an aggregate. pub(crate) fn agg_func_type(name: &str) -> i64 { - match name { - "COUNT" => AGG_COUNT, - "SUM" => AGG_SUM, - "AVG" => AGG_AVG, - "MIN" => AGG_MIN, - "MAX" => AGG_MAX, - _ => AGG_COUNT, // Default to COUNT - } + AggregateFunction::from_name(name) + .map(AggregateFunction::vm_type) + .unwrap_or(AGG_COUNT) } /// Patch a jump instruction's target address (p2) @@ -556,47 +633,6 @@ impl<'a> SqlCompiler<'a> { } } - /// Emit a comparison operation and return the result register - pub(crate) fn emit_comparison( - &mut self, - op: &BinaryOperator, - left_reg: i64, - right_reg: i64, - ) -> SqawkResult { - let result_reg = self.allocate_register(); - let opcode = Self::binary_op_to_comparison_opcode(op)?; - - self.emit( - opcode, - left_reg, - right_reg, - result_reg, - None, - &format!( - "r[{}] = (r[{}] {:?} r[{}])", - result_reg, left_reg, op, right_reg - ), - ); - - Ok(result_reg) - } - - /// Convert a comparison binary operator to the corresponding OpCode - pub(crate) fn binary_op_to_comparison_opcode(op: &BinaryOperator) -> SqawkResult { - match op { - BinaryOperator::Eq => Ok(OpCode::Eq), - BinaryOperator::NotEq => Ok(OpCode::Ne), - BinaryOperator::Lt => Ok(OpCode::Lt), - BinaryOperator::LtEq => Ok(OpCode::Le), - BinaryOperator::Gt => Ok(OpCode::Gt), - BinaryOperator::GtEq => Ok(OpCode::Ge), - _ => Err(SqawkError::UnsupportedSqlFeature(format!( - "Unsupported comparison operator: {:?}", - op - ))), - } - } - /// Convert SQL type to internal type string pub(crate) fn sql_type_to_internal(data_type: &str) -> &'static str { match data_type.to_uppercase().as_str() { @@ -607,50 +643,6 @@ impl<'a> SqlCompiler<'a> { } } - /// Find a column by table name and column name in a list of tables. - /// Returns (table_idx, col_idx) if found. - pub(crate) fn find_column_in_tables( - table_name: &str, - col_name: &str, - tables: &[&Table], - table_names: &[&str], - ) -> SqawkResult<(usize, usize)> { - for (i, name) in table_names.iter().enumerate() { - if table_name.eq_ignore_ascii_case(name) { - let columns = tables[i].columns(); - if let Some(col_idx) = columns - .iter() - .position(|c| c.eq_ignore_ascii_case(col_name)) - { - return Ok((i, col_idx)); - } else { - return Err(SqawkError::ColumnNotFound(col_name.to_string())); - } - } - } - Err(SqawkError::TableNotFound(table_name.to_string())) - } - - /// Build a wildcard schema (SELECT *) from multiple tables. - /// Returns column indices per table and the result schema. - pub(crate) fn build_wildcard_schema( - tables: &[&Table], - table_names: &[&str], - ) -> (Vec>, ResultSchema) { - let mut table_cols = Vec::with_capacity(tables.len()); - let mut schema = ResultSchema::new(); - - for (i, table) in tables.iter().enumerate() { - let cols: Vec = (0..table.column_count()).collect(); - for meta in table.column_metadata() { - schema.add_column(format!("{}.{}", table_names[i], meta.name), meta.data_type); - } - table_cols.push(cols); - } - - (table_cols, schema) - } - // Removed unused add_halt method /// Add a comment to the program if verbose mode is enabled @@ -676,6 +668,11 @@ impl<'a> SqlCompiler<'a> { self.label_addrs.clear(); self.pending_jumps.clear(); self.pending_jumps_three.clear(); + self.next_cursor = FIRST_TEMP_CURSOR; + self.next_sorter = 0; + self.outer_scope = None; + self.correlated.clear(); + self.insert_target = None; let init_addr = self.program.len(); self.emit(OpCode::Init, 0, 0, 0, None, "Start address filled in later"); @@ -766,6 +763,7 @@ impl<'a> SqlCompiler<'a> { match statement { Statement::Query(query) => self.compile_query(query), Statement::Insert(insert) => { + check_insert(insert)?; let source = insert.source.as_ref().ok_or_else(|| { SqawkError::UnsupportedSqlFeature("INSERT without a source".into()) })?; @@ -779,14 +777,19 @@ impl<'a> SqlCompiler<'a> { self.compile_insert(insert_target(&insert.table)?, &columns, source) } Statement::Delete(delete) => { + check_delete(delete)?; self.compile_delete(delete_from(&delete.from), delete.selection.as_ref()) } - Statement::Update(update) => self.compile_update( - &update.table, - &update.assignments, - update.selection.as_ref(), - ), + Statement::Update(update) => { + check_update(update)?; + self.compile_update( + &update.table, + &update.assignments, + update.selection.as_ref(), + ) + } Statement::CreateTable(create) => { + check_create_table(create)?; if let Some(q) = &create.query { // CREATE TABLE ... AS SELECT self.compile_create_table_as_select(&create.name, q) @@ -821,5001 +824,601 @@ impl<'a> SqlCompiler<'a> { "TRUNCATE of multiple tables is not supported".into(), )), }, - _ => Err(SqawkError::UnsupportedSqlFeature(format!( - "Unsupported SQL statement type: {:?}", - statement - ))), - } - } - - /// Compile a SQL query - fn compile_query(&mut self, query: &Query) -> SqawkResult<()> { - // `ORDER BY ALL` carries no key list, so it would otherwise look - // identical to "no ORDER BY" and be silently ignored. Reject it. - if is_order_by_all(query) { - return Err(SqawkError::UnsupportedSqlFeature( - "ORDER BY ALL is not supported".into(), - )); - } - - // Check if we have ORDER BY, LIMIT, or OFFSET - let has_order_by = !query_order_by(query).is_empty(); - let has_limit = query_limit(query).is_some() || query_offset(query).is_some(); - - // Compile the body (SELECT or set operation) - match &*query.body { - SetExpr::Select(select) => { - if has_order_by || has_limit { - self.compile_select_with_post_processing(select, query)?; - } else { - self.compile_select(select)?; - } - } - SetExpr::SetOperation { - op, - left, - right, - set_quantifier, - .. - } => { - let all = *set_quantifier == SetQuantifier::All; - self.compile_set_operation(op, left, right, all)?; - // TODO: Handle ORDER BY/LIMIT for set operations - } + // Name the statement by its leading keywords; the AST's Debug + // form used to be dumped here, hundreds of characters long. _ => { - return Err(SqawkError::UnsupportedSqlFeature( - "Only simple SELECT statements are supported".to_string(), - )); + let text = statement.to_string(); + let head: Vec<&str> = text.split_whitespace().take(2).collect(); + Err(SqawkError::UnsupportedSqlFeature(format!( + "{} statements are not supported", + head.join(" ") + ))) } } - - Ok(()) } - /// Compile a set operation (UNION, INTERSECT, EXCEPT) - fn compile_set_operation( - &mut self, - op: &SetOperator, - left: &SetExpr, - right: &SetExpr, - all: bool, - ) -> SqawkResult<()> { - self.add_comment(&format!( - "Set operation: {:?}{}", - op, - if all { " ALL" } else { "" } - )); - - // For INTERSECT and EXCEPT, we need to track where the left results end - let left_results_marker = match op { - SetOperator::Intersect | SetOperator::Except => { - // Use a register to store the count of left-side results - let marker_reg = self.allocate_register(); - Some(marker_reg) + /// Expand a SELECT list into one expression per output column. + /// + /// `*` and `t.*` become column references, so every output column -- + /// expanded or written -- is compiled by `code_expr` alike. + pub(crate) fn expand_projection( + &self, + projection: &[SelectItem], + plan: &FromPlan<'a>, + ) -> SqawkResult> { + let multi = plan.srcs.len() > 1; + let star_name = |refname: &str, col: &str| { + if multi && !plan.bare_star { + format!("{}.{}", refname, col) + } else { + col.to_string() } - _ => None, }; - - // Compile the left side - self.compile_set_expr(left)?; - - // For INTERSECT and EXCEPT, record the boundary - if let Some(marker_reg) = left_results_marker { - // Add a marker instruction that the VM will use to know where left results end - // p1=-1 is a special value meaning "use current result count" - // p2 is the destination register - self.emit( - OpCode::Integer, - -1, // Special marker: -1 means "current result count" - marker_reg, - 0, - None, - "Store current result count", - ); + let column = |refname: &str, col: &str| { + Expr::CompoundIdentifier(vec![Ident::new(refname), Ident::new(col)]) + }; + let mut outs = Vec::new(); + for item in projection { + match item { + SelectItem::Wildcard(_) => { + for (si, src) in plan.srcs.iter().enumerate() { + for (ci, meta) in src.table.column_metadata().iter().enumerate() { + if plan.is_hidden(si, ci) { + continue; + } + let expr = match plan.using_at(si, ci) { + Some(u) => Expr::Identifier(Ident::new(u.name.clone())), + None => column(&src.refname, &meta.name), + }; + outs.push(OutCol { + expr, + name: star_name(&src.refname, &meta.name), + alias: None, + }); + } + } + } + SelectItem::QualifiedWildcard(kind, _) => { + let qualifier = match kind { + SelectItemQualifiedWildcardKind::ObjectName(name) => { + object_name_last(name).to_ascii_lowercase() + } + SelectItemQualifiedWildcardKind::Expr(_) => { + return Err(SqawkError::UnsupportedSqlFeature( + "expr.* is not supported".into(), + )) + } + }; + let found = plan.srcs.iter().position(|s| { + s.refname == qualifier || s.table.name().eq_ignore_ascii_case(&qualifier) + }); + let src = match found { + Some(i) => &plan.srcs[i], + // A single table answers to its own name whatever + // alias it was given. + None if !multi => &plan.srcs[0], + None => return Err(SqawkError::TableNotFound(qualifier)), + }; + for meta in src.table.column_metadata() { + outs.push(OutCol { + expr: column(&src.refname, &meta.name), + name: star_name(&src.refname, &meta.name), + alias: None, + }); + } + } + SelectItem::UnnamedExpr(expr) => outs.push(OutCol { + expr: expr.clone(), + name: self.output_name(expr, multi), + alias: None, + }), + SelectItem::ExprWithAlias { expr, alias } => outs.push(OutCol { + expr: expr.clone(), + name: alias.value.clone(), + alias: Some(alias.value.clone()), + }), + // Multiple aliases for one item (`expr AS (a, b)`) is a + // non-standard extension sqawk does not implement. + SelectItem::ExprWithAliases { .. } => { + return Err(SqawkError::UnsupportedSqlFeature( + "Multiple column aliases are not supported".into(), + )) + } + } } + Ok(outs) + } - // Compile the right side - self.compile_set_expr(right)?; + /// Header for an unaliased SELECT-list expression. + /// + /// In a join a qualified column keeps its qualifier (`users.name`), as + /// join output always has; a single-table query shows the bare name. + pub(crate) fn output_name(&self, expr: &Expr, multi: bool) -> String { + match expr { + Expr::CompoundIdentifier(parts) if multi => parts + .iter() + .map(|p| p.value.as_str()) + .collect::>() + .join("."), + _ => self.get_column_name_from_expr(expr), + } + } - // Add the appropriate set operation opcode - match op { - SetOperator::Union => { - if !all { - // UNION (without ALL) needs to remove duplicates - self.emit( - OpCode::Distinct, - 0, - 0, - 0, - None, - "Remove duplicate rows for UNION", - ); - } - // UNION ALL: no additional processing needed, results are already combined + /// Emit bytecode to load a value as a literal + fn emit_value_literal(&mut self, reg: i64, value: &TableValue) -> SqawkResult<()> { + match value { + TableValue::Integer(i) => { + self.emit( + OpCode::Integer, + *i, + reg, + 0, + None, + &format!("r[{}] = {}", reg, i), + ); } - SetOperator::Intersect => { + TableValue::Float(f) => { + // Store float as string and use String opcode, then cast self.emit( - OpCode::Intersect, - left_results_marker.unwrap_or(0), + OpCode::String, 0, + reg, 0, - None, - "Keep only rows in both result sets", + Some(f.to_string()), + &format!("r[{}] = {}", reg, f), + ); + // Cast to float + self.emit( + OpCode::Cast, + reg, + reg, + 0, + Some("REAL".to_string()), + &format!("Cast r[{}] to REAL", reg), ); } - SetOperator::Except | SetOperator::Minus => { + TableValue::String(s) => { self.emit( - OpCode::Except, - left_results_marker.unwrap_or(0), + OpCode::String, 0, + reg, + 0, + Some(s.clone().into_owned()), + &format!("r[{}] = '{}'", reg, s), + ); + } + TableValue::Boolean(b) => { + self.emit( + OpCode::Integer, + if *b { 1 } else { 0 }, + reg, 0, None, - "Keep only rows in left but not in right", + &format!("r[{}] = {}", reg, b), ); } + TableValue::Null => { + self.emit(OpCode::Null, 0, reg, 0, None, &format!("r[{}] = NULL", reg)); + } } - Ok(()) } - /// Compile a set expression (can be a SELECT or nested set operation) - fn compile_set_expr(&mut self, expr: &SetExpr) -> SqawkResult<()> { - match expr { - SetExpr::Select(select) => self.compile_select(select), - SetExpr::SetOperation { - op, - left, - right, - set_quantifier, - .. - } => { - let all = *set_quantifier == SetQuantifier::All; - self.compile_set_operation(op, left, right, all) - } - _ => Err(SqawkError::UnsupportedSqlFeature( - "Unsupported set expression type".to_string(), - )), - } - } - - /// Rewrite `A INNER JOIN B ON cond` into `FROM A, B WHERE cond`. - /// - /// The implicit (comma) join path already supports aggregates and GROUP BY; - /// the explicit-JOIN path does not, and rejects `COUNT(*)` in its - /// projection resolver. For an INNER join the two spellings mean the same - /// thing, so an aggregate query over an explicit join is answered by - /// handing it to the machinery that can already do it, rather than by - /// duplicating that machinery. - /// - /// Returns `None` when the rewrite does not apply: outer joins are NOT - /// equivalent to a comma join plus WHERE, since the WHERE would discard - /// exactly the NULL-extended rows the outer join exists to produce. - fn rewrite_inner_join_as_implicit(select: &Select) -> Option