Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
42 changes: 40 additions & 2 deletions CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -53,10 +53,12 @@ add_library(svector SHARED
src/index/hnsw/graph.cc
src/index/hnsw/graph_ops.cc
src/index/hnsw/layer_ops.cc
src/index/hnsw/vector_cache.cc
src/storage/data_page.cc
src/storage/root_page.cc
src/storage/storage.cc
src/native_vector.cc
src/quantize.cc
src/vector.cc
)

Expand Down Expand Up @@ -130,10 +132,11 @@ target_compile_options(graph_ops_test PRIVATE -UNDEBUG)

add_test(NAME graph_ops_test COMMAND graph_ops_test)

# DistanceEvaluator test. Header-only like the graph/layer tests, but the
# evaluator takes Column::Data from the SDK, so it needs the SDK include dir.
# DistanceEvaluator test. The evaluator takes Column::Data from the SDK, so it
# needs the SDK include dir, and its quantized path calls quant::quantize().
add_executable(distance_evaluator_test
unittest/distance_evaluator_test.cc
src/quantize.cc
)

target_include_directories(distance_evaluator_test PRIVATE
Expand All @@ -144,6 +147,24 @@ target_compile_options(distance_evaluator_test PRIVATE -UNDEBUG)

add_test(NAME distance_evaluator_test COMMAND distance_evaluator_test)

# DecodedVectorCache test: the per-insert vector entries, including the seeded
# entry for the vector being inserted, and which kernel each pair of entries
# runs. Same SDK and quantize.cc needs as the DistanceEvaluator test, plus
# native_vector.cc for the decode each stored entry goes through.
add_executable(decoded_vector_cache_test
unittest/decoded_vector_cache_test.cc
src/native_vector.cc
src/quantize.cc
)

target_include_directories(decoded_vector_cache_test PRIVATE
${VillageSQLExtensionFramework_INCLUDE_DIR}
)

target_compile_options(decoded_vector_cache_test PRIVATE -UNDEBUG)

add_test(NAME decoded_vector_cache_test COMMAND decoded_vector_cache_test)

# Storage-layer test. Unlike the header-only graph/layer tests, it drives the
# real column-storage sources against an in-memory fake storage backend, so it
# compiles those sources and needs the SDK include dir (storage_api.h).
Expand All @@ -161,3 +182,20 @@ target_include_directories(storage_test PRIVATE
target_compile_options(storage_test PRIVATE -UNDEBUG)

add_test(NAME storage_test COMMAND storage_test)

# Vector-cache test. Compiles the cache and the quantization kernel it stores
# entries in; needs the SDK include dir for the Column/VID types the cache keys
# on.
add_executable(vector_cache_test
unittest/vector_cache_test.cc
src/index/hnsw/vector_cache.cc
src/quantize.cc
)

target_include_directories(vector_cache_test PRIVATE
${VillageSQLExtensionFramework_INCLUDE_DIR}
)

target_compile_options(vector_cache_test PRIVATE -UNDEBUG)

add_test(NAME vector_cache_test COMMAND vector_cache_test)
34 changes: 34 additions & 0 deletions mysql-test/r/hnsw_vector_cache_insert.result
Original file line number Diff line number Diff line change
@@ -0,0 +1,34 @@
SET PERSIST vsql_allow_preview_extensions = ON;
INSTALL EXTENSION vsql_vector;
SET @saved_cache_mode = @@GLOBAL.vsql_vector.cache_mode;
SET GLOBAL vsql_vector.cache_mode = 1;
CREATE TABLE t (
id INT PRIMARY KEY,
v SVECTOR(4) NOT NULL,
INDEX idx_v (v hnsw_l2) USING EXTENDED(hnsw) WITH (M = 2, ef_construction = 16)
) ENGINE=InnoDB;
SELECT VARIABLE_VALUE INTO @tomb0 FROM performance_schema.global_status
WHERE VARIABLE_NAME = 'vsql_vector.cache_tombstones';
SELECT VARIABLE_VALUE INTO @miss0 FROM performance_schema.global_status
WHERE VARIABLE_NAME = 'vsql_vector.cache_misses';
SELECT VARIABLE_VALUE INTO @tomb1 FROM performance_schema.global_status
WHERE VARIABLE_NAME = 'vsql_vector.cache_tombstones';
SELECT VARIABLE_VALUE INTO @miss1 FROM performance_schema.global_status
WHERE VARIABLE_NAME = 'vsql_vector.cache_misses';
SELECT COUNT(*) AS rows_inserted FROM t;
rows_inserted
300
# The cache was in use: the inserts' searches missed and filled it.
SELECT @miss1 - @miss0 > 0 AS cache_used;
cache_used
1
# No entry was retired: each new node was compared from its seeded entry,
# never admitted to the cache during its own insert.
SELECT @tomb1 - @tomb0 AS tombstones;
tombstones
0
DROP TABLE t;
SET GLOBAL vsql_vector.cache_mode = @saved_cache_mode;
UNINSTALL EXTENSION vsql_vector;
SET PERSIST vsql_allow_preview_extensions = OFF;
RESET PERSIST vsql_allow_preview_extensions;
66 changes: 66 additions & 0 deletions mysql-test/t/hnsw_vector_cache_insert.test
Original file line number Diff line number Diff line change
@@ -0,0 +1,66 @@
# Inserting into an HNSW index with the vector cache on must not retire cache
# entries.
#
# An insert compares its new node against neighbours whose lists it overflows
# (shrink_neighbours). If that comparison fetched the new node back by VID, the
# vector cache would admit it, and the invalidate at the end of insert() would
# then retire the entry it had just admitted: one dead slot per insert, which a
# fill-once cache never reclaims. create_node() instead seeds the insert's
# per-call vectors with the new node, so nothing is admitted during its own
# insert and the invalidate finds nothing live.
#
# M = 2 keeps neighbour lists short, so nearly every insert after the first few
# overflows some neighbour and takes the shrink path. The vectors are distinct
# so the build is not degenerate.

SET PERSIST vsql_allow_preview_extensions = ON;
INSTALL EXTENSION vsql_vector;

SET @saved_cache_mode = @@GLOBAL.vsql_vector.cache_mode;
SET GLOBAL vsql_vector.cache_mode = 1;

CREATE TABLE t (
id INT PRIMARY KEY,
v SVECTOR(4) NOT NULL,
INDEX idx_v (v hnsw_l2) USING EXTENDED(hnsw) WITH (M = 2, ef_construction = 16)
) ENGINE=InnoDB;

SELECT VARIABLE_VALUE INTO @tomb0 FROM performance_schema.global_status
WHERE VARIABLE_NAME = 'vsql_vector.cache_tombstones';
SELECT VARIABLE_VALUE INTO @miss0 FROM performance_schema.global_status
WHERE VARIABLE_NAME = 'vsql_vector.cache_misses';

# Each vector is passed to FROM_STRING as a literal: a computed string argument
# leaves the dimension unknown and the insert into SVECTOR(4) fails to decode.
--disable_query_log
--let $i = 1
while ($i <= 300)
{
--let $a = query_get_value(SELECT SIN($i) AS x, x, 1)
--let $b = query_get_value(SELECT COS($i) AS x, x, 1)
--let $c = query_get_value(SELECT SIN($i * 0.7) AS x, x, 1)
--let $d = query_get_value(SELECT COS($i * 1.3) AS x, x, 1)
--eval INSERT INTO t VALUES ($i, SVECTOR::FROM_STRING('[$a, $b, $c, $d]'))
--inc $i
}
--enable_query_log

SELECT VARIABLE_VALUE INTO @tomb1 FROM performance_schema.global_status
WHERE VARIABLE_NAME = 'vsql_vector.cache_tombstones';
SELECT VARIABLE_VALUE INTO @miss1 FROM performance_schema.global_status
WHERE VARIABLE_NAME = 'vsql_vector.cache_misses';

SELECT COUNT(*) AS rows_inserted FROM t;

--echo # The cache was in use: the inserts' searches missed and filled it.
SELECT @miss1 - @miss0 > 0 AS cache_used;

--echo # No entry was retired: each new node was compared from its seeded entry,
--echo # never admitted to the cache during its own insert.
SELECT @tomb1 - @tomb0 AS tombstones;

DROP TABLE t;
SET GLOBAL vsql_vector.cache_mode = @saved_cache_mode;
UNINSTALL EXTENSION vsql_vector;
SET PERSIST vsql_allow_preview_extensions = OFF;
RESET PERSIST vsql_allow_preview_extensions;
10 changes: 10 additions & 0 deletions src/distance_registry.h
Original file line number Diff line number Diff line change
Expand Up @@ -67,6 +67,16 @@ inline DistFn dist_for_name(const char *name) {
return nullptr;
}

// Whether this metric has a quantized (int16) kernel, and so whether an index
// on it can attach a vector cache. Only squared L2 today: quantize.h has
// dist_squared_l2_q and nothing else. Cosine is next and needs normalization
// at cache-fill time plus its own kernel; inner product needs a kernel; L1 has
// no precomputable per-vector reduction. The list lives here, next to the
// names, so adding a metric means touching one place.
inline bool has_quantized_kernel(const char *name) {
return name != nullptr && std::strcmp(name, kDistL2Squared) == 0;
}

} // namespace svector::native

#endif // VILLAGESQL_EXAMPLES_VSQL_SVECTOR_SRC_DISTANCE_REGISTRY_H
98 changes: 78 additions & 20 deletions src/index/hnsw/decoded_vector_cache.h
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,8 @@
#define VILLAGESQL_VSQL_VECTOR_SRC_INDEX_HNSW_DECODED_VECTOR_CACHE_H

#include "../../native_vector.h"
#include "../../quantize.h"
#include "distance_evaluator.h"
#include "hnsw.h"

#include <cstddef>
Expand All @@ -46,25 +48,78 @@ namespace svector::hnsw {
// vid's vector does not change while the call runs, so one decode serves
// every comparison in it.
//
// Entries live in fixed-size chunks that are never moved or freed until the
// cache is destroyed, so a pointer returned by find() or commit() stays valid
// for the cache's lifetime.
// On an index with a VectorCache attached, a node that cache holds is kept
// here as a pointer to the cache's entry, which is never rewritten while the
// cache lives, so nothing is copied. Only a node the cache refused, or every
// node on an index without one, gets a decoded copy.
//
// Bytes live in fixed-size chunks that are never moved or freed until the
// cache is destroyed, and entries in an unordered_map whose elements do not
// move, so pointers from find() and add() stay valid for the cache's lifetime.
class DecodedVectorCache {
public:
// entry_size is the most bytes one decoded vector can take; it is rounded
// up so every entry keeps native::Data's alignment.
// A node's vector in the forms resolved so far. f32 is set for a node the
// vector cache did not serve; q is the vector cache's entry for one it did,
// or a quantized copy of f32 made for a comparison against such a node.
struct Entry {
const native::Data *f32 = nullptr;
const quant::QData *q = nullptr;
};

// entry_size is the most bytes one decoded vector or quantized copy can
// take; it is rounded up so every entry keeps native::Data's alignment.
explicit DecodedVectorCache(size_t entry_size)
: m_entry_size(round_up(entry_size)) {}

// The cached decode of vid, or nullptr if it has none yet.
const native::Data *find(VID vid) const {
// vid's entry, or nullptr if it has none yet.
Entry *find(VID vid) {
auto it = m_entries.find(vid.value);
return it == m_entries.end() ? nullptr : it->second;
return it == m_entries.end() ? nullptr : &it->second;
}

// Records e as vid's entry.
Entry &add(VID vid, Entry e) { return m_entries.emplace(vid.value, e).first->second; }

// Records vid's entry from its raw stored value, decoded into the cache,
// plus q, its quantized form if the caller already has one (the vector being
// inserted, quantized once for its search). A no-op if vid already has an
// entry. out, if given, is pointed at vid's entry either way. Returns true
// on error.
bool add_stored(VID vid, const Column::Data &stored, const quant::QData *q,
DistanceEvaluator &dist, std::span<char> err,
Entry **out = nullptr) {
Entry *e = find(vid);
if (e == nullptr) {
const native::Data *decoded = nullptr;
if (dist.decode(stored, reserve(), entry_size(), &decoded, err))
return true;
commit();
e = &add(vid, {decoded, q});
}
if (out != nullptr)
*out = e;
return false;
}

// Bytes for the next entry, entry_size() long. Nothing is recorded until
// commit(), so a failed decode into them leaves the cache unchanged and the
// same bytes are handed out again.
// Distance between two entries. Two entries with an f32 form compare in
// f32, even if a mixed pair earlier gave one a quantized copy, so a pair's
// kernel never depends on what was compared before it. Otherwise at least
// one is held by the vector cache, which has no f32 form, so the other is
// brought to the quantized form too -- once, then kept in its entry.
// Returns true on error.
bool distance(Entry &a, Entry &b, DistanceEvaluator &dist, double &out,
std::span<char> err) {
if (a.f32 != nullptr && b.f32 != nullptr)
return dist.eval(a.f32, b.f32, out, err);
if ((a.q == nullptr && quantize(a, dist, err)) ||
(b.q == nullptr && quantize(b, dist, err)))
return true;
return dist.eval(a.q, b.q, out, err);
}

// Bytes for the next decoded vector or quantized copy, entry_size() long.
// They are not claimed until commit(), so a failed decode into them leaves
// the cache unchanged and the same bytes are handed out again.
void *reserve() {
if (m_used == m_chunks.size() * ENTRIES_PER_CHUNK) {
m_chunks.emplace_back(new std::byte[m_entry_size * ENTRIES_PER_CHUNK]);
Expand All @@ -73,18 +128,21 @@ class DecodedVectorCache {
(m_used % ENTRIES_PER_CHUNK) * m_entry_size;
}

// Records the bytes the last reserve() returned, now holding vid's decoded
// vector.
const native::Data *commit(VID vid) {
const auto *data = static_cast<const native::Data *>(reserve());
m_entries.emplace(vid.value, data);
++m_used;
return data;
}
// Claims the bytes the last reserve() returned.
void commit() { ++m_used; }

size_t entry_size() const { return m_entry_size; }

private:
// Gives e, which must have an f32 form, a quantized copy in the cache.
bool quantize(Entry &e, DistanceEvaluator &dist, std::span<char> err) {
if (dist.quantize(e.f32->data, e.f32->dim, reserve(), entry_size(), &e.q,
err))
return true;
commit();
return false;
}

static constexpr size_t ENTRIES_PER_CHUNK = 64;

static size_t round_up(size_t n) {
Expand All @@ -97,7 +155,7 @@ class DecodedVectorCache {
// new std::byte[] is aligned to max_align_t, which native::length() asks
// for, and m_entry_size keeps every entry after the first aligned too.
std::vector<std::unique_ptr<std::byte[]>> m_chunks;
std::unordered_map<uint64_t, const native::Data *> m_entries;
std::unordered_map<uint64_t, Entry> m_entries;
};

} // namespace svector::hnsw
Expand Down
Loading
Loading