Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
11 changes: 11 additions & 0 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -80,6 +80,17 @@ python3 skills/img-gen-prompts/scripts/oip.py search \
--limit 5
```

Search keeps exact matches in `results`. When exact coverage is sparse, a
separate `related_results` channel may provide image-confirmed references that
miss exactly one declared aesthetic preference. It never adds a vector
database, model download, API key, or Python dependency. Run the labeled,
bilingual 72-query regression benchmark (including visually reviewed related
references) with:

```bash
npm run test:retrieval
```

On Windows, replace `python3` with `py -3` or use the Skill through a compatible Agent.

## Public data boundary
Expand Down
9 changes: 9 additions & 0 deletions README.zh-CN.md
Original file line number Diff line number Diff line change
Expand Up @@ -72,6 +72,15 @@ npm run status

准备完成后会返回 `"active_taxonomy_version": "oip-visual-v2"` 和 `"ready": true`。

搜索结果中的 `results` 始终是严格匹配。如果严格结果不足,独立的
`related_results` 会补充“仅缺少一个已声明审美偏好”的图片侧确认参考,
不会把它冒充严格命中。这项能力不增加向量数据库、模型下载、API Key 或
Python 依赖。可以运行包含相关图片人工视觉判定的 72 条中英双语标注回归基准:

```bash
npm run test:retrieval
```

## 公开数据边界

公开 DB 只保留产品运行所需数据:
Expand Down
36 changes: 36 additions & 0 deletions evals/retrieval/intents.jsonl
Original file line number Diff line number Diff line change
@@ -0,0 +1,36 @@
{"id":"R01","intent":{"zh-Hans":"雨夜霓虹小巷里的电影感女性人像","en":"Cinematic female portrait in a neon rainy alley"},"must_tags_all":["subject_type:portrait","visual_style:cinematic","lighting:neon-light","scene:alleyway"],"forbidden_tags":["visual_style:black-and-white"]}
{"id":"R02","intent":{"zh-Hans":"白底运动鞋电商产品图","en":"White-background e-commerce sneaker product shot"},"must_tags_all":["subject_type:product","visual_style:product-photography","color_palette:white-minimal"],"forbidden_tags":["quality_flags:logo-watermark","quality_flags:embedded-text"]}
{"id":"R03","intent":{"zh-Hans":"暗色奢华香水广告","en":"Dark luxury perfume advertisement"},"must_tags_all":["subject_type:product","visual_style:product-photography","color_palette:dark-moody","mood:luxurious"],"forbidden_tags":["mood:playful"]}
{"id":"R04","intent":{"zh-Hans":"咖啡馆里的生活方式咖啡照片","en":"Lifestyle coffee image inside a cafe"},"must_tags_all":["subject_type:food-drink","visual_style:lifestyle","scene:cafe-restaurant"],"forbidden_tags":["scene:studio"]}
{"id":"R05","intent":{"zh-Hans":"街头仰拍的全身时尚大片","en":"Low-angle full-body street fashion editorial"},"must_tags_all":["subject_type:fashion-person","visual_style:editorial","composition:full-body","composition:low-angle"],"forbidden_tags":["subject_type:beauty-closeup"]}
{"id":"R06","intent":{"zh-Hans":"森林里的水彩儿童角色","en":"Watercolor children's character in a forest"},"must_tags_all":["subject_type:character","visual_style:watercolor","scene:forest"],"forbidden_tags":["visual_style:photography"]}
{"id":"R07","intent":{"zh-Hans":"像素风太空角色游戏素材","en":"Pixel-art space character game asset"},"must_tags_all":["subject_type:character","visual_style:pixel-art"],"forbidden_tags":["visual_style:photography"]}
{"id":"R08","intent":{"zh-Hans":"粉彩黏土玩具","en":"Playful pastel clay toy"},"must_tags_all":["visual_style:clay-render","color_palette:pastel","mood:playful"],"forbidden_tags":["color_palette:dark-moody"],"related_relevant_ids":{"zh-Hans":["2045789509850153418","2056153760540328167"],"en":["2045789509850153418","2031792275765047546"]}}
{"id":"R09","intent":{"zh-Hans":"复古漫画动作海报","en":"Retro comic action poster"},"must_tags_all":["visual_style:comic","visual_style:retro","usage:poster"],"forbidden_tags":["visual_style:photography"]}
{"id":"R10","intent":{"zh-Hans":"极简留白建筑主视觉","en":"Minimal architecture image with generous negative space"},"must_tags_all":["subject_type:architecture-interior","visual_style:minimal","composition:negative-space"],"forbidden_tags":["quality_flags:typography-heavy"]}
{"id":"R11","intent":{"zh-Hans":"有胡须老人的黑白近景肖像","en":"Black-and-white close portrait of an older bearded man"},"must_tags_all":["subject_type:portrait","visual_style:black-and-white","composition:close-up","human_attributes:facial-hair"],"forbidden_tags":["color_palette:vibrant"]}
{"id":"R12","intent":{"zh-Hans":"高调影棚里的美妆特写","en":"High-key studio beauty close-up"},"must_tags_all":["subject_type:beauty-closeup","lighting:studio-light","lighting:high-key"],"forbidden_tags":["composition:wide-shot"]}
{"id":"R13","intent":{"zh-Hans":"蓝调时刻的电影感汽车","en":"Cinematic car at blue hour"},"must_tags_all":["subject_type:vehicle","visual_style:cinematic","lighting:blue-hour"],"forbidden_tags":["visual_style:illustration"]}
{"id":"R14","intent":{"zh-Hans":"明亮的美食俯拍","en":"Bright overhead food flat lay"},"must_tags_all":["subject_type:food-drink","composition:overhead-flatlay","lighting:high-key"],"forbidden_tags":["lighting:low-key"]}
{"id":"R15","intent":{"zh-Hans":"水下超现实时装大片","en":"Surreal underwater fashion editorial"},"must_tags_all":["subject_type:fashion-person","visual_style:surreal","scene:underwater"],"forbidden_tags":["scene:indoor-home"]}
{"id":"R16","intent":{"zh-Hans":"沙漠黄金时刻的自然风景","en":"Desert landscape at golden hour"},"must_tags_all":["subject_type:landscape-nature","scene:desert","lighting:golden-hour"],"forbidden_tags":["scene:studio"]}
{"id":"R17","intent":{"zh-Hans":"科幻室内里的未来角色","en":"Futuristic character inside a sci-fi interior"},"must_tags_all":["subject_type:character","scene:sci-fi-interior","mood:futuristic"],"forbidden_tags":["scene:historical-setting"]}
{"id":"R18","intent":{"zh-Hans":"日式花园油画","en":"Oil painting of a Japanese garden"},"must_tags_all":["subject_type:landscape-nature","visual_style:oil-painting","scene:garden"],"forbidden_tags":["visual_style:photography"]}
{"id":"R19","intent":{"zh-Hans":"SaaS 控制台 UI 截图","en":"SaaS dashboard UI screen"},"must_tags_all":["subject_type:ui-screen","quality_flags:screenshot-ui"],"forbidden_tags":["visual_style:photography"]}
{"id":"R20","intent":{"zh-Hans":"演示文稿里的信息图","en":"Infographic for a presentation"},"must_tags_all":["subject_type:diagram-document","usage:presentation"],"forbidden_tags":["subject_type:portrait"]}
{"id":"R21","intent":{"zh-Hans":"霓虹抽象平面封面","en":"Neon abstract graphic cover"},"must_tags_all":["subject_type:abstract","visual_style:graphic-design","color_palette:neon"],"forbidden_tags":["subject_type:portrait"],"related_relevant_ids":{"en":["2076663782554697935"]}}
{"id":"R22","intent":{"zh-Hans":"历史场景中的复古胶片情侣","en":"Vintage-film couple in a historical setting"},"must_tags_all":["subject_type:couple","visual_style:film-vintage","scene:historical-setting"],"forbidden_tags":["mood:futuristic"]}
{"id":"R23","intent":{"zh-Hans":"充满活力的运动群像","en":"Energetic group sports scene"},"must_tags_all":["subject_type:group","scene:sports","mood:energetic"],"forbidden_tags":["mood:calm"]}
{"id":"R24","intent":{"zh-Hans":"温馨家居中的宠物生活照","en":"Cozy lifestyle pet image at home"},"must_tags_all":["subject_type:animal","visual_style:lifestyle","scene:indoor-home","mood:cozy"],"forbidden_tags":["scene:studio"]}
{"id":"R25","intent":{"zh-Hans":"奢华珠宝微距产品图","en":"Luxury macro jewelry product image"},"must_tags_all":["subject_type:product","camera_lens:macro","mood:luxurious"],"forbidden_tags":["composition:wide-shot"]}
{"id":"R26","intent":{"zh-Hans":"黑白摄影中的可见纹身","en":"Visible tattoo in black-and-white photography"},"must_tags_all":["human_attributes:tattoo-visible","visual_style:photography","color_palette:black-and-white"],"forbidden_tags":["visual_style:illustration"]}
{"id":"R27","intent":{"zh-Hans":"可用于品牌的极简线稿矢量图","en":"Minimal line-art vector for brand use"},"must_tags_all":["subject_type:graphic-design","visual_style:line-art","workflow:vector-generation"],"forbidden_tags":["visual_style:photography"]}
{"id":"R28","intent":{"zh-Hans":"烛光里的浪漫人像","en":"Romantic portrait by candlelight"},"must_tags_all":["subject_type:portrait","lighting:candlelight","mood:romantic"],"forbidden_tags":["lighting:high-key"]}
{"id":"R29","intent":{"zh-Hans":"霓虹科技产品 3D 广告","en":"Neon 3D tech-product advertisement"},"must_tags_all":["subject_type:product","visual_style:3d-render","lighting:neon-light","usage:ad-campaign"],"forbidden_tags":["visual_style:film-vintage"]}
{"id":"R30","intent":{"zh-Hans":"黑白低调光的小巷黑色电影人像","en":"Black-and-white low-key noir alley portrait"},"must_tags_all":["subject_type:portrait","visual_style:black-and-white","scene:alleyway","lighting:low-key"],"forbidden_tags":["lighting:high-key"]}
{"id":"R31","intent":{"zh-Hans":"阳光海滩上的时装人物","en":"Fashion subject on a daylight beach"},"must_tags_all":["subject_type:fashion-person","scene:beach","lighting:natural-daylight"],"forbidden_tags":["scene:studio"]}
{"id":"R32","intent":{"zh-Hans":"城市航拍长曝光感觉","en":"Aerial city view with a long-exposure feeling"},"must_tags_all":["subject_type:city-street","composition:aerial-view"],"forbidden_tags":["composition:close-up"],"related_relevant_ids":{"zh-Hans":["2033247548572684433","2077314594159169745","2079065900716544405","2062553617522106535","2065623612170911996"]}}
{"id":"R33","intent":{"zh-Hans":"等距视角的 3D 办公空间","en":"Isometric 3D office workspace"},"must_tags_all":["subject_type:architecture-interior","visual_style:3d-render","camera_lens:isometric","scene:office-workspace"],"forbidden_tags":["visual_style:photography"]}
{"id":"R34","intent":{"zh-Hans":"巨型物体出现在城市里的超现实远景","en":"Surreal wide city scene with a gigantic object"},"must_tags_all":["subject_type:city-street","visual_style:surreal","composition:wide-shot"],"forbidden_tags":["visual_style:minimal"]}
{"id":"R35","intent":{"zh-Hans":"带包装文字的产品平面广告","en":"Product graphic ad with packaging typography"},"must_tags_all":["subject_type:product","visual_style:graphic-design","quality_flags:embedded-text"],"forbidden_tags":["quality_flags:screenshot-ui"]}
{"id":"R36","intent":{"zh-Hans":"大字排版和留白的极简海报","en":"Minimal poster with large typography and whitespace"},"must_tags_all":["subject_type:graphic-design","visual_style:minimal","composition:negative-space","quality_flags:typography-heavy"],"forbidden_tags":["composition:multi-panel"]}
204 changes: 204 additions & 0 deletions evals/retrieval/run_benchmark.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,204 @@
#!/usr/bin/env python3
"""Deterministic bilingual benchmark for exact and related retrieval channels.

Exact relevance is checked against labeled constraints. Related relevance is
fail-closed against image IDs that were manually reviewed for the query.
"""
from __future__ import annotations

import argparse
import json
import statistics
import sys
import time
from pathlib import Path
from types import SimpleNamespace
from typing import Any

REPO_DIR = Path(__file__).resolve().parents[2]
if str(REPO_DIR) not in sys.path:
sys.path.insert(0, str(REPO_DIR))

from retrieval.engine import RETRIEVAL_VERSION, parse_intent
from runtime.archive_db import DEFAULT_DB_PATH, connect_read_only
from scripts import prompt_library


def load_cases(path: Path) -> list[dict[str, Any]]:
return [
json.loads(line)
for line in path.read_text(encoding="utf-8").splitlines()
if line.strip()
]


def percentile(values: list[float], position: float) -> float:
if not values:
return 0.0
ordered = sorted(values)
return ordered[min(len(ordered) - 1, int((len(ordered) - 1) * position))]


def tag_ids(item: dict[str, Any]) -> set[str]:
return {str(tag["id"]) for tag in item.get("tags") or []}


def main() -> int:
parser = argparse.ArgumentParser()
parser.add_argument("--db", type=Path, default=DEFAULT_DB_PATH)
parser.add_argument("--intents", type=Path, default=Path(__file__).with_name("intents.jsonl"))
parser.add_argument("--out", type=Path)
parser.add_argument("--limit", type=int, default=5)
args = parser.parse_args()

details = []
durations = []
strict_relevant = strict_returned = 0
related_relevant = related_returned = 0
strict_capacity = 0
parser_expected = parser_found = 0
zero_exact_queries = zero_queries_recovered = 0
queries_with_related = related_hard_violations = related_unjudged = 0
locale_strict_relevant = {"zh-Hans": 0, "en": 0}
locale_capacity = {"zh-Hans": 0, "en": 0}

with connect_read_only(args.db) as conn:
for case in load_cases(args.intents):
for locale, query in case["intent"].items():
expected = set(case["must_tags_all"])
forbidden = set(case["forbidden_tags"])
intent = parse_intent(query)
parsed_tags = {
item.tag for item in [*intent.locked_tags, *intent.must_tags]
}
parser_expected += len(expected)
parser_found += len(expected & parsed_tags)

started = time.perf_counter()
result = prompt_library.run_search(
conn,
SimpleNamespace(
query=query,
tag=None,
author=None,
tool=None,
limit=args.limit,
max_prompt_chars=400,
max_tags=100,
taxonomy=RETRIEVAL_VERSION,
),
)
durations.append(time.perf_counter() - started)

exact_rows = []
for item in result["results"]:
tags = tag_ids(item)
relevant = expected.issubset(tags) and not (forbidden & tags)
strict_relevant += int(relevant)
locale_strict_relevant[locale] += int(relevant)
exact_rows.append({
"tweet_id": item["tweet_id"],
"relevant": relevant,
})

related_rows = []
reviewed_related_ids = set(
(case.get("related_relevant_ids") or {}).get(locale) or []
)
for item in result["related_results"]:
tags = tag_ids(item)
missing = expected - tags
hard_violations = sorted(forbidden & tags)
reviewed_relevant = item["tweet_id"] in reviewed_related_ids
unjudged = not reviewed_relevant
relevant = reviewed_relevant and len(missing) <= 1 and not hard_violations
related_relevant += int(relevant)
related_unjudged += int(unjudged)
related_hard_violations += len(hard_violations)
related_rows.append({
"tweet_id": item["tweet_id"],
"relevant": relevant,
"visually_reviewed": reviewed_relevant,
"missing_expected": sorted(missing),
"declared_missing": item["missing_constraints"],
"hard_violations": hard_violations,
})

exact_count = len(exact_rows)
related_count = len(related_rows)
strict_returned += exact_count
related_returned += related_count
strict_capacity += args.limit
locale_capacity[locale] += args.limit
zero_exact_queries += int(exact_count == 0)
zero_queries_recovered += int(exact_count == 0 and related_count > 0)
queries_with_related += int(related_count > 0)
details.append({
"id": case["id"],
"locale": locale,
"query": query,
"exact": exact_rows,
"related": related_rows,
})

strict_precision = strict_relevant / strict_returned if strict_returned else 0.0
related_precision = related_relevant / related_returned if related_returned else 1.0
strict_yield = strict_relevant / strict_capacity if strict_capacity else 0.0
combined_yield = (
(strict_relevant + related_relevant) / strict_capacity
if strict_capacity else 0.0
)
summary = {
"schema_version": "oip-retrieval-benchmark-v2",
"taxonomy_version": RETRIEVAL_VERSION,
"queries": len(details),
"parser_must_coverage": round(parser_found / parser_expected, 4),
"strict": {
"precision_on_returned": round(strict_precision, 4),
"yield_adjusted_p_at_5": round(strict_yield, 4),
"average_results": round(strict_returned / len(details), 2),
"zero_result_queries": zero_exact_queries,
},
"related": {
"human_judged_precision": round(related_precision, 4),
"results": related_returned,
"queries_supplemented": queries_with_related,
"zero_queries_recovered": zero_queries_recovered,
"unjudged_results": related_unjudged,
"hard_violations": related_hard_violations,
},
"combined_useful_at_5": round(combined_yield, 4),
"relative_useful_gain": (
round((combined_yield - strict_yield) / strict_yield, 4)
if strict_yield else 0.0
),
"locale_strict_p_at_5": {
locale: round(locale_strict_relevant[locale] / locale_capacity[locale], 4)
for locale in locale_capacity
},
"latency_ms": {
"median": round(statistics.median(durations) * 1000, 2),
"p95": round(percentile(durations, 0.95) * 1000, 2),
"max": round(max(durations) * 1000, 2),
},
}
report = {"summary": summary, "details": details}
rendered = json.dumps(report, ensure_ascii=False, indent=2) + "\n"
if args.out:
args.out.parent.mkdir(parents=True, exist_ok=True)
args.out.write_text(rendered, encoding="utf-8")
print(json.dumps(summary, ensure_ascii=False, indent=2))

passed = (
summary["parser_must_coverage"] == 1.0
and strict_precision >= 0.93
and related_precision >= 0.95
and related_unjudged == 0
and related_hard_violations == 0
and combined_yield > strict_yield
)
return 0 if passed else 1


if __name__ == "__main__":
raise SystemExit(main())
3 changes: 2 additions & 1 deletion package.json
Original file line number Diff line number Diff line change
Expand Up @@ -12,8 +12,9 @@
"data:pull": "git lfs pull",
"dev": "npm --prefix web run dev",
"status": "node scripts/run_python.mjs skills/img-gen-prompts/scripts/oip.py status",
"test": "npm run verify:data && npm run test:api && npm --prefix web test",
"test": "npm run verify:data && npm run test:api && npm run test:retrieval && npm --prefix web test",
"test:api": "node scripts/run_python.mjs server/test_oip_api.py",
"test:retrieval": "node scripts/run_python.mjs -m unittest tests/test_related_retrieval.py && node scripts/run_python.mjs evals/retrieval/run_benchmark.py",
"verify:data": "node scripts/run_python.mjs scripts/verify_public_data.py",
"lint": "npm --prefix web run lint",
"build": "npm --prefix web run build",
Expand Down
Loading
Loading