juiceb0xc0de commited on
Commit
8887de2
Β·
verified Β·
1 Parent(s): d2ea1eb

B-Sides app code (indexes load from dataset repos at boot)

Browse files
.DS_Store ADDED
Binary file (10.2 kB). View file
 
.pytest_cache/.gitignore ADDED
@@ -0,0 +1,2 @@
 
 
 
1
+ # Created by pytest automatically.
2
+ *
.pytest_cache/CACHEDIR.TAG ADDED
@@ -0,0 +1,4 @@
 
 
 
 
 
1
+ Signature: 8a477f597d28d172789f06886806bc55
2
+ # This file is a cache directory tag created by pytest.
3
+ # For information about cache directory tags, see:
4
+ # https://bford.info/cachedir/spec.html
.pytest_cache/README.md ADDED
@@ -0,0 +1,8 @@
 
 
 
 
 
 
 
 
 
1
+ # pytest cache directory #
2
+
3
+ This directory contains data from the pytest's cache plugin,
4
+ which provides the `--lf` and `--ff` options, as well as the `cache` fixture.
5
+
6
+ **Do not** commit this to version control.
7
+
8
+ See [the docs](https://docs.pytest.org/en/stable/how-to/cache.html) for more information.
.pytest_cache/v/cache/nodeids ADDED
@@ -0,0 +1,117 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ [
2
+ "tests/test_config.py::test_collection_validation_names_missing_credential[env0-HF_TOKEN]",
3
+ "tests/test_config.py::test_collection_validation_names_missing_credential[env1-OPENAI_API_KEY]",
4
+ "tests/test_config.py::test_crawl_end_is_an_explicit_inclusive_upper_boundary",
5
+ "tests/test_config.py::test_defaults_match_the_production_crawl_contract",
6
+ "tests/test_config.py::test_environment_overrides_are_parsed_without_storing_secret_values",
7
+ "tests/test_config.py::test_invalid_settings_are_rejected[env0-BSIDES_MODE]",
8
+ "tests/test_config.py::test_invalid_settings_are_rejected[env1-Hub request budget]",
9
+ "tests/test_config.py::test_invalid_settings_are_rejected[env2-Hub request budget]",
10
+ "tests/test_config.py::test_invalid_settings_are_rejected[env3-embedding dimensions]",
11
+ "tests/test_config.py::test_invalid_settings_are_rejected[env4-overlap]",
12
+ "tests/test_config.py::test_invalid_settings_are_rejected[env5-queue multiplier]",
13
+ "tests/test_config.py::test_invalid_settings_are_rejected[env6-embedding workers]",
14
+ "tests/test_config.py::test_invalid_settings_are_rejected[env7-embedding token rate]",
15
+ "tests/test_config.py::test_invalid_settings_are_rejected[env8-checkpoint interval]",
16
+ "tests/test_config.py::test_invalid_settings_are_rejected[env9-progress interval]",
17
+ "tests/test_config.py::test_search_mode_does_not_require_collection_credentials",
18
+ "tests/test_runtime.py::test_app_registers_docker_corpus_configuration",
19
+ "tests/test_runtime.py::test_app_registers_kernel_corpus_configuration",
20
+ "tests/test_runtime.py::test_docker_facets_and_search_use_container_response_shape",
21
+ "tests/test_runtime.py::test_docker_index_download_allows_only_published_search_files",
22
+ "tests/test_runtime.py::test_docker_semantic_search_without_a_key_is_unavailable_not_a_crash",
23
+ "tests/test_runtime.py::test_endpoints_503_until_the_index_is_ready",
24
+ "tests/test_runtime.py::test_facets_and_picks_still_answer",
25
+ "tests/test_runtime.py::test_failed_docker_load_does_not_affect_model_search",
26
+ "tests/test_runtime.py::test_failed_kernel_load_does_not_affect_model_search",
27
+ "tests/test_runtime.py::test_frontend_checks_model_exclusions_by_default",
28
+ "tests/test_runtime.py::test_frontend_defaults_docker_to_recipes_with_an_archive_toggle",
29
+ "tests/test_runtime.py::test_frontend_escapes_every_docker_card_value",
30
+ "tests/test_runtime.py::test_frontend_keeps_model_size_as_precise_billion_boxes",
31
+ "tests/test_runtime.py::test_frontend_links_docker_artifacts_to_valid_hugging_face_targets",
32
+ "tests/test_runtime.py::test_frontend_offers_all_three_corpora",
33
+ "tests/test_runtime.py::test_frontend_restores_default_exclusions_when_clearing_the_deck",
34
+ "tests/test_runtime.py::test_frontend_routes_every_corpus_through_one_url_builder",
35
+ "tests/test_runtime.py::test_frontend_tracks_docker_in_every_corpus_state_map",
36
+ "tests/test_runtime.py::test_kernel_facets_and_search_use_kernel_response_shape",
37
+ "tests/test_runtime.py::test_model_search_exclusions_are_explicit_and_api_compatible",
38
+ "tests/test_runtime.py::test_models_client_remains_the_oai_compatibility_shim",
39
+ "tests/test_runtime.py::test_reload_endpoint_hot_swaps_only_the_docker_corpus",
40
+ "tests/test_runtime.py::test_reload_endpoint_hot_swaps_only_the_kernel_corpus",
41
+ "tests/test_runtime.py::test_reload_of_an_unknown_corpus_is_a_404",
42
+ "tests/test_runtime.py::test_reload_with_a_wrong_token_is_403",
43
+ "tests/test_runtime.py::test_reload_with_an_empty_body_is_rejected",
44
+ "tests/test_runtime.py::test_reload_with_the_right_token_swaps_the_index",
45
+ "tests/test_runtime.py::test_reload_without_a_configured_token_is_503",
46
+ "tests/test_runtime.py::test_search_defaults_to_the_models_corpus",
47
+ "tests/test_runtime.py::test_search_index_download_excludes_crawler_staging",
48
+ "tests/test_runtime.py::test_search_query_uses_the_loaded_index_dimensions",
49
+ "tests/test_runtime.py::test_search_response_keeps_the_total_models_envelope",
50
+ "tests/test_runtime.py::test_semantic_search_without_a_key_is_unavailable_not_a_crash",
51
+ "tests/test_runtime.py::test_semantic_searches_use_their_assigned_corpus_clients",
52
+ "tests/test_runtime.py::test_unknown_corpus_is_a_404",
53
+ "tests/test_search_indexes.py::test_build_where_covers_every_model_filter",
54
+ "tests/test_search_indexes.py::test_build_where_empty_request_is_neutral",
55
+ "tests/test_search_indexes.py::test_build_where_executes_against_real_sqlite",
56
+ "tests/test_search_indexes.py::test_card_decodes_json_columns_and_rounds_score",
57
+ "tests/test_search_indexes.py::test_card_without_score_omits_it",
58
+ "tests/test_search_indexes.py::test_compute_facets_shape_matches_the_legacy_payload",
59
+ "tests/test_search_indexes.py::test_container_archive_rows_appear_when_artifact_kind_filter_is_absent",
60
+ "tests/test_search_indexes.py::test_container_card_decodes_json_arrays_and_preserves_empty_arrays",
61
+ "tests/test_search_indexes.py::test_container_defaults_to_requested_recipe_filter",
62
+ "tests/test_search_indexes.py::test_container_facets_count_scalars_arrays_and_booleans",
63
+ "tests/test_search_indexes.py::test_container_filters_are_anded_across_groups_and_ored_within_group",
64
+ "tests/test_search_indexes.py::test_container_query_text_is_preserved_for_semantic_search",
65
+ "tests/test_search_indexes.py::test_container_scalar_boolean_and_json_filters_execute_against_sqlite",
66
+ "tests/test_search_indexes.py::test_db_is_thread_local_readonly_and_generation_scoped",
67
+ "tests/test_search_indexes.py::test_download_index_requests_dataset_files_into_spec_dir",
68
+ "tests/test_search_indexes.py::test_embed_query_caps_input_at_max_chars",
69
+ "tests/test_search_indexes.py::test_embed_query_uses_spec_model_and_selected_matrix_dimensions",
70
+ "tests/test_search_indexes.py::test_embed_query_without_client_names_the_selected_credential",
71
+ "tests/test_search_indexes.py::test_embed_query_without_matrix_is_unavailable",
72
+ "tests/test_search_indexes.py::test_extract_constraints_empty_remainder_skips_embedding",
73
+ "tests/test_search_indexes.py::test_extract_constraints_never_loosens_explicit_filters",
74
+ "tests/test_search_indexes.py::test_extract_constraints_pulls_dates_params_downloads_and_flags",
75
+ "tests/test_search_indexes.py::test_failed_download_keeps_serving_the_previous_index",
76
+ "tests/test_search_indexes.py::test_kernel_adapter_ignores_legacy_model_and_unknown_filters",
77
+ "tests/test_search_indexes.py::test_kernel_array_filter_values_are_or_within_one_category",
78
+ "tests/test_search_indexes.py::test_kernel_card_decodes_arrays_and_returns_facts",
79
+ "tests/test_search_indexes.py::test_kernel_facets_count_members_of_json_arrays",
80
+ "tests/test_search_indexes.py::test_kernel_filters_execute_against_real_sqlite",
81
+ "tests/test_search_indexes.py::test_kernel_query_text_is_preserved_for_semantic_search",
82
+ "tests/test_search_indexes.py::test_kernel_structured_page_uses_kernel_envelope_contract",
83
+ "tests/test_search_indexes.py::test_load_accepts_empty_index",
84
+ "tests/test_search_indexes.py::test_load_isolates_downloader_failure",
85
+ "tests/test_search_indexes.py::test_load_pulls_dataset_when_files_are_missing",
86
+ "tests/test_search_indexes.py::test_load_rejects_dimension_mismatch",
87
+ "tests/test_search_indexes.py::test_load_rejects_matrix_rank_not_two",
88
+ "tests/test_search_indexes.py::test_load_rejects_non_finite_values",
89
+ "tests/test_search_indexes.py::test_load_rejects_row_count_mismatch",
90
+ "tests/test_search_indexes.py::test_load_rejects_rows_outside_matrix_bounds",
91
+ "tests/test_search_indexes.py::test_load_rejects_sqlite_that_fails_quick_check",
92
+ "tests/test_search_indexes.py::test_load_validates_marks_ready_and_computes_facets",
93
+ "tests/test_search_indexes.py::test_load_without_downloader_never_imports_hub_when_files_exist",
94
+ "tests/test_search_indexes.py::test_malformed_config_remains_searchable_during_quantization_exclusion",
95
+ "tests/test_search_indexes.py::test_model_adapter_exposes_spec_result_key_and_card_columns",
96
+ "tests/test_search_indexes.py::test_model_adapter_ignores_adapter_filters",
97
+ "tests/test_search_indexes.py::test_model_exclusions_are_neutral_by_default",
98
+ "tests/test_search_indexes.py::test_model_exclusions_match_family_evidence",
99
+ "tests/test_search_indexes.py::test_model_family_exclusion_matches_expanded_mistral_terms",
100
+ "tests/test_search_indexes.py::test_quantization_exclusion_requires_strong_evidence",
101
+ "tests/test_search_indexes.py::test_registry_isolates_a_broken_corpus_from_a_healthy_one",
102
+ "tests/test_search_indexes.py::test_registry_start_loading_runs_isolated_threads",
103
+ "tests/test_search_indexes.py::test_registry_unknown_key_raises_keyerror",
104
+ "tests/test_search_indexes.py::test_reload_does_not_skip_the_download_step",
105
+ "tests/test_search_indexes.py::test_reload_invalidates_connections_to_the_replaced_file",
106
+ "tests/test_search_indexes.py::test_reload_rejecting_a_bad_publish_keeps_serving_the_old_index",
107
+ "tests/test_search_indexes.py::test_reload_rejects_a_publish_whose_rows_do_not_match_the_matrix",
108
+ "tests/test_search_indexes.py::test_reload_swaps_in_the_freshly_published_index",
109
+ "tests/test_search_indexes.py::test_search_request_keeps_optional_adapter_filters",
110
+ "tests/test_search_indexes.py::test_semantic_rank_empty_candidates_skip_embedding",
111
+ "tests/test_search_indexes.py::test_semantic_rank_page_beyond_window_returns_total_but_no_cards",
112
+ "tests/test_search_indexes.py::test_semantic_rank_propagates_unavailable_embedding",
113
+ "tests/test_search_indexes.py::test_semantic_rank_scores_orders_and_paginates",
114
+ "tests/test_search_indexes.py::test_semantic_rank_uses_candidate_scoring_for_small_filtered_sets",
115
+ "tests/test_search_indexes.py::test_structured_page_orders_counts_and_paginates",
116
+ "tests/test_search_indexes.py::test_structured_page_unknown_sort_falls_back_to_newest"
117
+ ]
Dockerfile ADDED
@@ -0,0 +1,22 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ FROM python:3.12-slim
2
+
3
+ WORKDIR /app
4
+ ENV BSIDES_MODE=search
5
+ # git isn't in the slim base, but HF's Space builder injects a
6
+ # `git config --global user.email ...` step after our layers β€” it 127s without git.
7
+ RUN apt-get update && apt-get install -y --no-install-recommends git \
8
+ && rm -rf /var/lib/apt/lists/*
9
+ COPY requirements.txt .
10
+ RUN pip install --no-cache-dir -r requirements.txt
11
+
12
+ # Retrieval only: the collector and its job scripts are not part of this image.
13
+ COPY app.py staff_picks.json ./
14
+ COPY bsides/__init__.py bsides/config.py bsides/search_indexes.py bsides/
15
+ COPY static static
16
+
17
+ # index is pulled from the b-sides-index dataset at boot (see app.py)
18
+ RUN mkdir -p /app/index && chmod 777 /app/index
19
+ RUN mkdir -p /tmp/bsides && chmod 777 /tmp/bsides
20
+
21
+ EXPOSE 7860
22
+ CMD ["uvicorn", "app:app", "--host", "0.0.0.0", "--port", "7860"]
README.md CHANGED
@@ -1,11 +1,62 @@
1
  ---
2
- title: B Sides V2
3
- emoji: πŸ‘
4
- colorFrom: pink
5
- colorTo: gray
6
  sdk: docker
 
7
  pinned: false
8
- license: mit
9
  ---
10
 
11
- Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
  ---
2
+ title: B-Sides V2
3
+ emoji: πŸ”Ž
4
+ colorFrom: red
5
+ colorTo: yellow
6
  sdk: docker
7
+ app_port: 7860
8
  pinned: false
9
+ short_description: Building live to let the obscure breathe new life
10
  ---
11
 
12
+ # πŸ”Ž B-Sides V2 β€” the full text-model record
13
+
14
+ B-Sides V2 is a fresh, searchable full-history pressing of Hugging Face text
15
+ models.
16
+
17
+ B-Sides index is more interesting than "models nobody uses." It maps the shadow
18
+ infrastructure of open ML: abandoned experiments, regional language work,
19
+ architecture probes, creative utilities, and research releases that conventional
20
+ popularity rankings erase.
21
+
22
+ The production collector walks both official Hub pipeline streams from newest
23
+ to oldest until they are exhausted:
24
+
25
+ - `text-generation`
26
+ - `text2text-generation`
27
+
28
+ It deduplicates full repository IDs. There are no family, popularity, download,
29
+ parameter, likes, license, gating, private-visibility, or source-content
30
+ exclusions: every discovered model receives a metadata embedding, and available
31
+ cards/configuration/source are added to it.
32
+
33
+ For every survivor, B-Sides fetches the root model card plus every Python file at
34
+ the discovered commit SHA, chunks them by source lines, and sends deterministic
35
+ requests to `text-embedding-3-small` through the OpenAI Batch API. The collector
36
+ checkpoints its SQLite state to
37
+ [`juiceb0xc0de/b-sides-v2-index`](https://huggingface.co/datasets/juiceb0xc0de/b-sides-v2-index),
38
+ resumes after restarts, and moves into overlapping tail scans after the initial
39
+ backfill.
40
+
41
+ All Hub traffic shares a conservative governor: at most 2,400 counted or
42
+ reserved requests per 300 seconds against the confirmed 3,000-request hard cap.
43
+
44
+ The existing search index remains online while this full-history pressing is
45
+ built. New search activation stays gated until coverage, chunk/vector alignment,
46
+ dimensions, finite values, and checksums validate.
47
+
48
+ Collector progress is available at `/api/crawl-status`.
49
+
50
+ ## Runtime configuration
51
+
52
+ - `BSIDES_MODE=search|collect|both` (default: `both`)
53
+ - `BSIDES_DATASET` (default: `juiceb0xc0de/b-sides-v2-index`)
54
+ - `BSIDES_RUN_ID` (default: `v2`)
55
+ - `BSIDES_CRAWL_END` (optional ISO-8601 exclusive upper boundary)
56
+ - `BSIDES_HUB_REQUESTS_PER_5M` (default: `2400`, must stay below `3000`)
57
+ - `BSIDES_CHECKPOINT_EVERY_REPOS` (default: `2500`)
58
+ - `BSIDES_TAIL_INTERVAL_SECONDS` (default: `900`)
59
+ - `BSIDES_TAIL_OVERLAP_IDS` (default: `100`)
60
+ - Space secrets: `HF_TOKEN` and `OPENAI_API_KEY`
61
+
62
+ Pressed by [juiceb0xc0de](https://huggingface.co/juiceb0xc0de).
__pycache__/app.cpython-311.pyc ADDED
Binary file (12.8 kB). View file
 
__pycache__/app.cpython-313.pyc ADDED
Binary file (11.6 kB). View file
 
__pycache__/build_index.cpython-311.pyc ADDED
Binary file (21.1 kB). View file
 
__pycache__/build_index.cpython-314.pyc ADDED
Binary file (20.2 kB). View file
 
__pycache__/rebuild_index_repo.cpython-314.pyc ADDED
Binary file (10.8 kB). View file
 
__pycache__/run_container_job.cpython-311.pyc ADDED
Binary file (6.4 kB). View file
 
__pycache__/run_container_job.cpython-314.pyc ADDED
Binary file (4.06 kB). View file
 
__pycache__/run_hf_job.cpython-311.pyc ADDED
Binary file (12.2 kB). View file
 
app.py ADDED
@@ -0,0 +1,265 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """
2
+ B-Sides β€” deep search for indie & vintage text-generation LLMs.
3
+
4
+ FastAPI backend. A thin HTTP layer over `bsides.search_indexes`: the registry
5
+ owns downloading, loading, validation, facets and ranking for every corpus
6
+ (models, kernels and Docker images). This module owns the wire shape and
7
+ nothing else.
8
+
9
+ Retrieval only. Collection lives in its own job repos: this Space reads the
10
+ published indexes and never crawls, so nothing here imports the collector.
11
+
12
+ Endpoints:
13
+ GET / static frontend
14
+ GET /api/facets filter options + counts for one corpus
15
+ POST /api/search semantic + structured search over one corpus
16
+ POST /api/reload token-gated hot-swap of a freshly published index
17
+ GET /api/picks the daily staff-pick spread (models corpus)
18
+ """
19
+
20
+ import hmac
21
+ import os
22
+ import threading
23
+ from contextlib import asynccontextmanager
24
+ from pathlib import Path
25
+
26
+ import orjson
27
+ from fastapi import FastAPI, HTTPException
28
+ from fastapi.responses import FileResponse, Response
29
+ from openai import OpenAI
30
+ from pydantic import BaseModel
31
+
32
+ from bsides.config import Settings
33
+ from bsides.search_indexes import (
34
+ PAGE_SIZE,
35
+ ContainerAdapter,
36
+ IndexRegistry,
37
+ IndexSpec,
38
+ KernelAdapter,
39
+ ModelAdapter,
40
+ SearchReq,
41
+ UnavailableError,
42
+ semantic_rank,
43
+ structured_page,
44
+ )
45
+
46
+ BASE = Path(__file__).parent
47
+ INDEX_DIR = Path(os.environ.get('BSIDES_INDEX', BASE / 'index'))
48
+ INDEX_DATASET = os.environ.get('BSIDES_DATASET', 'juiceb0xc0de/b-sides-v2-index')
49
+ KERNEL_DATASET = os.environ.get(
50
+ 'BSIDES_KERNEL_DATASET', 'juiceb0xc0de/b-sides-v2-kernels')
51
+ CONTAINER_DATASET = os.environ.get(
52
+ 'BSIDES_CONTAINER_DATASET', 'juiceb0xc0de/b-sides-v2-containers')
53
+
54
+ EMBED_MODEL = 'text-embedding-3-small' # must match the corpus embedding model
55
+ MODELS = 'models' # default corpus for every endpoint
56
+ KERNELS = 'kernels'
57
+ DOCKER = 'docker'
58
+ QUERY_CREDENTIALS = {
59
+ MODELS: 'OPENAI_API_KEY',
60
+ KERNELS: 'OPENAI_KERNEL_API_KEY',
61
+ DOCKER: 'OPENAI_DOCKER_API_KEY',
62
+ }
63
+
64
+ # The published matrix width is a property of the crawl that produced it, so the
65
+ # spec's expected width comes from the same Settings the collector embeds with.
66
+ # It is a validation expectation only β€” the width actually sent to the embedding
67
+ # API is read off the loaded matrix (see search_indexes.embed_query).
68
+ EMBED_DIMENSIONS = Settings.from_env(os.environ).embedding_dimensions
69
+
70
+
71
+ @asynccontextmanager
72
+ async def _lifespan(_application: FastAPI):
73
+ _startup()
74
+ yield
75
+
76
+
77
+ app = FastAPI(title='B-Sides', lifespan=_lifespan)
78
+
79
+
80
+ def _openai_client(env_name: str):
81
+ key = os.environ.get(env_name)
82
+ return OpenAI(api_key=key) if key else None
83
+
84
+
85
+ query_clients = {
86
+ corpus: _openai_client(credential)
87
+ for corpus, credential in QUERY_CREDENTIALS.items()
88
+ }
89
+ oai = query_clients[MODELS]
90
+
91
+ # ── corpora ───────────────────────────────────────────────────────────────────
92
+ # Registration is import-time and cheap; the 1.4 GB matrix β€” and on a cold boot
93
+ # the ~800 MB index download β€” load in background threads kicked off by the
94
+ # lifespan startup. That is the whole point: uvicorn binds :7860 immediately, so
95
+ # HF's Space watchdog stops re-spawning duplicate workers while the matrix is
96
+ # still streaming off disk. Until an index is ready its endpoints 503.
97
+ MODEL_SPEC = IndexSpec(
98
+ key=MODELS,
99
+ dataset_id=INDEX_DATASET,
100
+ local_dir=INDEX_DIR,
101
+ table='models',
102
+ id_column='model_id',
103
+ embed_model=EMBED_MODEL,
104
+ dimensions=EMBED_DIMENSIONS,
105
+ )
106
+ KERNEL_SPEC = IndexSpec(
107
+ key=KERNELS,
108
+ dataset_id=KERNEL_DATASET,
109
+ local_dir=INDEX_DIR / KERNELS,
110
+ table='kernels',
111
+ id_column='repo_id',
112
+ embed_model=EMBED_MODEL,
113
+ dimensions=256,
114
+ )
115
+ DOCKER_SPEC = IndexSpec(
116
+ key=DOCKER,
117
+ dataset_id=CONTAINER_DATASET,
118
+ local_dir=INDEX_DIR / DOCKER,
119
+ table='containers',
120
+ id_column='repo_id',
121
+ embed_model=EMBED_MODEL,
122
+ dimensions=256,
123
+ )
124
+
125
+ registry = IndexRegistry()
126
+ registry.register(ModelAdapter(MODEL_SPEC))
127
+ registry.register(KernelAdapter(KERNEL_SPEC))
128
+ registry.register(ContainerAdapter(DOCKER_SPEC))
129
+
130
+
131
+ def _ready_index(corpus: str):
132
+ """Resolve a corpus key to a loaded index, or fail the way HTTP expects."""
133
+ try:
134
+ index = registry.index(corpus)
135
+ except KeyError:
136
+ raise HTTPException(404, f'unknown corpus: {corpus}') from None
137
+ if not index.ready.is_set():
138
+ # `error` set means load() failed and recorded why β€” that state is
139
+ # permanent until the artifact is fixed, so "retry in a moment" would
140
+ # be a lie. (Reload failures keep ready set and never land here.)
141
+ if index.error is not None:
142
+ raise HTTPException(
143
+ 503, f'index failed to load: {index.error}')
144
+ raise HTTPException(503, 'index still loading β€” retry in a moment')
145
+ return index
146
+
147
+
148
+ def _startup():
149
+ # bind first, load in the background β€” see the note above MODEL_SPEC.
150
+ registry.start_loading()
151
+ print('startup: :7860 binding while the index loads in the background',
152
+ flush=True)
153
+
154
+
155
+ # ── search ────────────────────────────────────────────────────────────────────
156
+ @app.post('/api/search')
157
+ def search(r: SearchReq, corpus: str = MODELS):
158
+ index = _ready_index(corpus)
159
+ adapter = index.adapter
160
+ # Fill the facet slots the user left empty from the query text, then filter.
161
+ # Order matters: extraction mutates the request that build_where reads.
162
+ sem_q = adapter.extract_constraints(r)
163
+ where, args = adapter.build_where(r)
164
+ try:
165
+ if sem_q and r.sort == 'relevance':
166
+ total, cards = semantic_rank(
167
+ index.db(), index, adapter, where=where, args=args,
168
+ sem_q=sem_q, page=r.page, page_size=PAGE_SIZE,
169
+ embed_client=query_clients[corpus],
170
+ credential_name=QUERY_CREDENTIALS[corpus])
171
+ else:
172
+ total, cards = structured_page(
173
+ index.db(), index, adapter, where=where, args=args,
174
+ sort=r.sort, page=r.page, page_size=PAGE_SIZE)
175
+ except UnavailableError as e:
176
+ raise HTTPException(503, str(e)) from e
177
+ return Response(orjson.dumps({'total': total, adapter.result_key: cards}),
178
+ media_type='application/json')
179
+
180
+
181
+ @app.get('/api/facets')
182
+ def facets(corpus: str = MODELS):
183
+ return Response(orjson.dumps(_ready_index(corpus).facets),
184
+ media_type='application/json')
185
+
186
+
187
+ # ── reload ────────────────────────────────────────────────────────────────────
188
+ # publish_loop.py POSTs here after every publish so the Space picks up the new
189
+ # index without a restart. The token gate matters: a reload checks large index
190
+ # artifacts on the Hub and can transfer changed files, so an open endpoint would
191
+ # let anyone burn our bandwidth on demand.
192
+ RELOAD_TOKEN = os.environ.get('BSIDES_RELOAD_TOKEN', '')
193
+ _reload_lock = threading.Lock()
194
+
195
+
196
+ class ReloadReq(BaseModel):
197
+ token: str = ''
198
+
199
+
200
+ @app.post('/api/reload')
201
+ def reload_corpus(body: ReloadReq, corpus: str = MODELS):
202
+ """Kick off a hot-swap and answer immediately.
203
+
204
+ The download is the slow part and the caller times out at 60 s, so the
205
+ swap runs on a background thread; `LoadedIndex.reload` keeps the current
206
+ index serving until the new one has validated.
207
+ """
208
+ if not RELOAD_TOKEN:
209
+ raise HTTPException(
210
+ 503, 'reload not configured: set BSIDES_RELOAD_TOKEN')
211
+ if not hmac.compare_digest(body.token, RELOAD_TOKEN):
212
+ raise HTTPException(403, 'reload token does not match')
213
+ try:
214
+ index = registry.index(corpus)
215
+ except KeyError:
216
+ raise HTTPException(404, f'unknown corpus: {corpus}') from None
217
+ # Non-blocking: a publish that overlaps the previous one is told to wait
218
+ # rather than racing a second download into the same directory.
219
+ if not _reload_lock.acquire(blocking=False):
220
+ raise HTTPException(409, 'a reload is already running')
221
+
222
+ def run() -> None:
223
+ try:
224
+ index.reload(registry.downloader)
225
+ finally:
226
+ _reload_lock.release()
227
+
228
+ threading.Thread(target=run, name=f'bsides-reload-{corpus}',
229
+ daemon=True).start()
230
+ return {'status': 'reload started', 'corpus': corpus}
231
+
232
+
233
+ SPREAD_SIZE = 4
234
+
235
+ @app.get('/api/picks')
236
+ def picks():
237
+ """The daily spread: SPREAD_SIZE cards drawn from the staff_picks.json pool,
238
+ rotated deterministically by UTC date β€” no state, fresh every midnight."""
239
+ index = _ready_index(MODELS)
240
+ adapter = index.adapter
241
+ picks_path = BASE / 'staff_picks.json'
242
+ if not picks_path.exists():
243
+ return Response(orjson.dumps({'picks': []}), media_type='application/json')
244
+ pool = orjson.loads(picks_path.read_bytes())
245
+ if len(pool) > SPREAD_SIZE:
246
+ import datetime, random
247
+ today = datetime.datetime.now(datetime.timezone.utc).strftime('%Y-%m-%d')
248
+ pool = random.Random(today).sample(pool, SPREAD_SIZE)
249
+ c = index.db()
250
+ out = []
251
+ for e in pool:
252
+ row = c.execute(
253
+ f'SELECT {adapter.card_columns} FROM {index.spec.table} '
254
+ f'WHERE {index.spec.id_column} = ?', (e['model_id'],)).fetchone()
255
+ if row:
256
+ d = adapter.card(row)
257
+ d['note'] = e.get('note', '')
258
+ d['picked_by'] = e.get('picked_by', 'staff')
259
+ out.append(d)
260
+ return Response(orjson.dumps({'picks': out}), media_type='application/json')
261
+
262
+
263
+ @app.get('/')
264
+ def home():
265
+ return FileResponse(BASE / 'static' / 'index.html')
bsides/.DS_Store ADDED
Binary file (6.15 kB). View file
 
bsides/__init__.py ADDED
@@ -0,0 +1,5 @@
 
 
 
 
 
 
1
+ """B-Sides collection and semantic-index pipeline."""
2
+
3
+ __all__ = ["__version__"]
4
+
5
+ __version__ = "0.1.0"
bsides/config.py ADDED
@@ -0,0 +1,202 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """Validated runtime settings for the B-Sides collector."""
2
+
3
+ from __future__ import annotations
4
+
5
+ from dataclasses import dataclass
6
+ from pathlib import Path
7
+ from typing import Mapping
8
+
9
+
10
+ PIPELINES = (
11
+ "text-generation",
12
+ "text2text-generation",
13
+ )
14
+ # V2 maps every model in its official text streams. Retained as an empty
15
+ # compatibility export for existing downstream sample tooling.
16
+ BLACKLIST: tuple[str, ...] = ()
17
+
18
+ # Listing query parameter per pipeline. The Hub's pipeline_tag=listings for
19
+ # text2text-generation began answering HTTP 200 with an empty page (confirmed
20
+ # 2026-08-30 in the b-sides-v2-index-2025 audit): the crawler read that as
21
+ # stream exhaustion and dropped ~8,300 models. filter= does a tag match and
22
+ # still enumerates the stream; the extra dual-tagged rows land under
23
+ # text-generation too and dedupe on repo id.
24
+ LISTING_PARAMS = {
25
+ "text-generation": "pipeline_tag",
26
+ "text2text-generation": "filter",
27
+ }
28
+
29
+
30
+ def _int(env: Mapping[str, str], name: str, default: int) -> int:
31
+ try:
32
+ return int(env.get(name, str(default)))
33
+ except ValueError as exc:
34
+ raise ValueError(f"{name} must be an integer") from exc
35
+
36
+
37
+ def _float(env: Mapping[str, str], name: str, default: float) -> float:
38
+ try:
39
+ return float(env.get(name, str(default)))
40
+ except ValueError as exc:
41
+ raise ValueError(f"{name} must be a number") from exc
42
+
43
+
44
+ @dataclass(frozen=True, slots=True)
45
+ class Settings:
46
+ pipelines: tuple[str, ...] = PIPELINES
47
+ blacklist: tuple[str, ...] = BLACKLIST
48
+ mode: str = "both"
49
+ dataset_repo_id: str = "juiceb0xc0de/b-sides-v2-index"
50
+ run_id: str = "v2"
51
+ crawl_year: int | None = None
52
+ crawl_end: str | None = None
53
+ state_dir: Path = Path("/tmp/bsides")
54
+ hub_requests_per_window: int = 2_400
55
+ # Zero observes resolver traffic without proactively throttling it. Public
56
+ # model file fetches are not consuming the account's resolver allowance;
57
+ # real 429 responses are retried by HubGateway instead.
58
+ hub_resolver_requests_per_window: int = 0
59
+ hub_window_seconds: float = 300.0
60
+ # Fetching is latency-bound, so admitted repositories resolve concurrently.
61
+ fetch_workers: int = 24
62
+ fetch_queue_multiplier: int = 4
63
+ embed_workers: int = 8
64
+ checkpoint_interval_seconds: float = 1_800.0
65
+ progress_interval_seconds: float = 10.0
66
+ embedding_model: str = "text-embedding-3-small"
67
+ embedding_dimensions: int = 256
68
+ embedding_tokens_per_minute: int = 4_500_000
69
+ chunk_target_tokens: int = 1_200
70
+ chunk_overlap_tokens: int = 150
71
+ max_file_bytes: int = 2 * 1024 * 1024
72
+ max_repo_bytes: int = 16 * 1024 * 1024
73
+ checkpoint_every_repos: int = 2_500
74
+ tail_interval_seconds: float = 900.0
75
+ tail_overlap_ids: int = 100
76
+ lease_ttl_seconds: float = 1_800.0
77
+ hf_token_present: bool = False
78
+ openai_api_key_present: bool = False
79
+
80
+ @classmethod
81
+ def from_env(cls, env: Mapping[str, str]) -> "Settings":
82
+ settings = cls(
83
+ mode=env.get("BSIDES_MODE", "both").strip().lower(),
84
+ dataset_repo_id=env.get(
85
+ "BSIDES_DATASET", "juiceb0xc0de/b-sides-v2-index"
86
+ ).strip(),
87
+ run_id=env.get("BSIDES_RUN_ID", "v2").strip(),
88
+ crawl_year=(
89
+ _int(env, "BSIDES_YEAR", 0) if env.get("BSIDES_YEAR", "").strip() else None
90
+ ),
91
+ crawl_end=env.get("BSIDES_CRAWL_END", "").strip() or None,
92
+ state_dir=Path(env.get("BSIDES_STATE_DIR", "/tmp/bsides")),
93
+ hub_requests_per_window=_int(
94
+ env, "BSIDES_HUB_REQUESTS_PER_5M", 2_400
95
+ ),
96
+ hub_resolver_requests_per_window=_int(
97
+ env, "BSIDES_HUB_RESOLVER_REQUESTS_PER_5M", 0
98
+ ),
99
+ fetch_workers=_int(env, "BSIDES_FETCH_WORKERS", 24),
100
+ fetch_queue_multiplier=_int(
101
+ env, "BSIDES_FETCH_QUEUE_MULTIPLIER", 4
102
+ ),
103
+ embed_workers=_int(env, "BSIDES_EMBED_WORKERS", 8),
104
+ checkpoint_interval_seconds=_float(
105
+ env, "BSIDES_CHECKPOINT_INTERVAL_SECONDS", 1_800.0
106
+ ),
107
+ progress_interval_seconds=_float(
108
+ env, "BSIDES_PROGRESS_INTERVAL_SECONDS", 10.0
109
+ ),
110
+ embedding_dimensions=_int(
111
+ env, "BSIDES_EMBEDDING_DIMENSIONS", 256
112
+ ),
113
+ embedding_tokens_per_minute=_int(
114
+ env, "BSIDES_EMBEDDING_TPM", 4_500_000
115
+ ),
116
+ chunk_target_tokens=_int(
117
+ env, "BSIDES_CHUNK_TARGET_TOKENS", 1_200
118
+ ),
119
+ chunk_overlap_tokens=_int(
120
+ env, "BSIDES_CHUNK_OVERLAP_TOKENS", 150
121
+ ),
122
+ max_file_bytes=_int(env, "BSIDES_MAX_FILE_BYTES", 2 * 1024 * 1024),
123
+ max_repo_bytes=_int(
124
+ env, "BSIDES_MAX_REPO_BYTES", 16 * 1024 * 1024
125
+ ),
126
+ checkpoint_every_repos=_int(
127
+ env, "BSIDES_CHECKPOINT_EVERY_REPOS", 2_500
128
+ ),
129
+ tail_interval_seconds=_float(
130
+ env, "BSIDES_TAIL_INTERVAL_SECONDS", 900.0
131
+ ),
132
+ tail_overlap_ids=_int(env, "BSIDES_TAIL_OVERLAP_IDS", 100),
133
+ lease_ttl_seconds=_float(
134
+ env, "BSIDES_LEASE_TTL_SECONDS", 1_800.0
135
+ ),
136
+ hf_token_present=bool(env.get("HF_TOKEN", "").strip()),
137
+ openai_api_key_present=bool(env.get("OPENAI_API_KEY", "").strip()),
138
+ )
139
+ settings._validate()
140
+ return settings
141
+
142
+ def _validate(self) -> None:
143
+ if self.mode not in {"search", "collect", "both"}:
144
+ raise ValueError("BSIDES_MODE must be search, collect, or both")
145
+ if not 1 <= self.hub_requests_per_window < 3_000:
146
+ raise ValueError("Hub request budget must be between 1 and 2999")
147
+ if not 0 <= self.hub_resolver_requests_per_window < 20_000:
148
+ raise ValueError(
149
+ "Resolver budget must be between 0 and 19999"
150
+ )
151
+ if not 1 <= self.fetch_workers <= 64:
152
+ raise ValueError("fetch workers must be between 1 and 64")
153
+ if self.fetch_queue_multiplier <= 0:
154
+ raise ValueError("fetch queue multiplier must be positive")
155
+ if self.embed_workers <= 0:
156
+ raise ValueError("embedding workers must be positive")
157
+ if self.checkpoint_interval_seconds <= 0:
158
+ raise ValueError("checkpoint interval must be positive")
159
+ if self.progress_interval_seconds <= 0:
160
+ raise ValueError("progress interval must be positive")
161
+ if self.embedding_dimensions <= 0:
162
+ raise ValueError("embedding dimensions must be positive")
163
+ if self.embedding_tokens_per_minute <= 0:
164
+ raise ValueError("embedding token rate must be positive")
165
+ if self.chunk_target_tokens <= 0:
166
+ raise ValueError("chunk target must be positive")
167
+ if not 0 <= self.chunk_overlap_tokens < self.chunk_target_tokens:
168
+ raise ValueError("chunk overlap must be smaller than the target")
169
+ if self.max_file_bytes <= 0 or self.max_repo_bytes < self.max_file_bytes:
170
+ raise ValueError("artifact byte limits are inconsistent")
171
+ if self.checkpoint_every_repos <= 0:
172
+ raise ValueError("checkpoint interval must be positive")
173
+ if self.tail_interval_seconds <= 0:
174
+ raise ValueError("tail interval must be positive")
175
+ if self.tail_overlap_ids <= 0:
176
+ raise ValueError("tail overlap must be positive")
177
+ if self.lease_ttl_seconds <= 0:
178
+ raise ValueError("lease TTL must be positive")
179
+ if not self.dataset_repo_id:
180
+ raise ValueError("BSIDES_DATASET must not be empty")
181
+ if not self.run_id:
182
+ raise ValueError("BSIDES_RUN_ID must not be empty")
183
+ if self.crawl_year is not None and not 2000 <= self.crawl_year <= 9999:
184
+ raise ValueError("BSIDES_YEAR must be between 2000 and 9999")
185
+ if self.crawl_end is not None:
186
+ from datetime import datetime
187
+
188
+ try:
189
+ datetime.fromisoformat(self.crawl_end.replace("Z", "+00:00"))
190
+ except ValueError as exc:
191
+ raise ValueError("BSIDES_CRAWL_END must be an ISO-8601 timestamp") from exc
192
+
193
+ def validate_for_collection(self) -> None:
194
+ if self.mode == "search":
195
+ return
196
+ missing = []
197
+ if not self.hf_token_present:
198
+ missing.append("HF_TOKEN")
199
+ if not self.openai_api_key_present:
200
+ missing.append("OPENAI_API_KEY")
201
+ if missing:
202
+ raise ValueError("missing collection credentials: " + ", ".join(missing))
bsides/search_indexes.py ADDED
@@ -0,0 +1,1003 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """Reusable multi-corpus search index core.
2
+
3
+ A corpus (models today; kernels and Docker images next) is one normalized
4
+ embedding matrix plus one SQLite metadata table whose `row` column aligns
5
+ 1:1 with matrix rows. This module owns everything schema-agnostic:
6
+
7
+ IndexSpec static corpus definition (dataset, table, embedding)
8
+ LoadedIndex runtime state: matrix, readiness, errors, connections
9
+ IndexRegistry registration, isolated loading, per-index status
10
+ embed_query query embedding against one index's model/dimensions
11
+ semantic_rank candidate rows β†’ embed β†’ score β†’ top-k page
12
+ structured_page COUNT + ORDER BY paging without embeddings
13
+
14
+ Schema knowledge lives in CorpusAdapter implementations β€” ModelAdapter
15
+ reproduces today's model behavior verbatim. app.py stays a thin HTTP
16
+ layer over the registry.
17
+
18
+ Loader contract: `LoadedIndex.load` never raises. A corpus that fails to
19
+ download or validate records its error and stays not-ready; the other
20
+ corpora keep serving.
21
+ """
22
+
23
+ from __future__ import annotations
24
+
25
+ import re
26
+ import sqlite3
27
+ import threading
28
+ from dataclasses import dataclass
29
+ from pathlib import Path
30
+ from typing import Protocol
31
+
32
+ import numpy as np
33
+ import orjson
34
+ from pydantic import BaseModel, Field
35
+
36
+ MATRIX_NAME = 'embeddings.f16.npy'
37
+ SQLITE_NAME = 'meta.sqlite'
38
+ DEFAULT_FILES = (MATRIX_NAME, SQLITE_NAME)
39
+
40
+ MAX_QUERY_CHARS = 300
41
+ PAGE_SIZE = 24
42
+
43
+
44
+ class UnavailableError(RuntimeError):
45
+ """A search dependency is missing (embedding client, loaded index).
46
+
47
+ Raised from the core; the HTTP layer maps it to 503 with the message.
48
+ """
49
+
50
+
51
+ @dataclass(frozen=True)
52
+ class IndexSpec:
53
+ """Static definition of one searchable corpus."""
54
+ key: str # registry key: 'models' | 'kernels' | 'docker'
55
+ dataset_id: str # HF dataset repo publishing the index files
56
+ local_dir: Path # where the corpus files live on disk
57
+ table: str # SQLite table holding the metadata rows
58
+ id_column: str # human id column: 'model_id' | 'repo_id' | ...
59
+ embed_model: str # embedding model used for queries
60
+ dimensions: int # expected vector width; validated at load
61
+ files: tuple[str, ...] = DEFAULT_FILES
62
+
63
+
64
+
65
+
66
+ def _validate_matrix(spec: IndexSpec, matrix: np.ndarray) -> None:
67
+ if matrix.ndim != 2:
68
+ raise ValueError(
69
+ f'{spec.key}: matrix rank {matrix.ndim} != 2')
70
+ if matrix.shape[1] != spec.dimensions:
71
+ raise ValueError(
72
+ f'{spec.key}: matrix has {matrix.shape[1]} dimensions, '
73
+ f'spec expects {spec.dimensions}')
74
+ if not np.isfinite(matrix).all():
75
+ raise ValueError(f'{spec.key}: matrix contains non-finite values')
76
+
77
+
78
+ def _validate_sqlite(spec: IndexSpec, conn: sqlite3.Connection,
79
+ matrix_rows: int) -> None:
80
+ quick = conn.execute('PRAGMA quick_check').fetchone()[0]
81
+ if quick != 'ok':
82
+ raise ValueError(f'{spec.key}: sqlite quick_check failed: {quick}')
83
+ count = conn.execute(
84
+ f'SELECT COUNT(*) FROM {spec.table}').fetchone()[0]
85
+ if count != matrix_rows:
86
+ raise ValueError(
87
+ f'{spec.key}: {count} rows in {spec.table} vs '
88
+ f'{matrix_rows} matrix rows')
89
+ if count:
90
+ lo, hi = conn.execute(
91
+ f'SELECT MIN(row), MAX(row) FROM {spec.table}').fetchone()
92
+ if lo < 0 or hi >= matrix_rows:
93
+ raise ValueError(
94
+ f'{spec.key}: row ids {lo}..{hi} outside matrix bounds '
95
+ f'0..{matrix_rows - 1}')
96
+
97
+
98
+ class LoadedIndex:
99
+ """Runtime state for one corpus: matrix, readiness, connections."""
100
+
101
+ def __init__(self, spec: IndexSpec, adapter=None) -> None:
102
+ self.spec = spec
103
+ self.adapter = adapter
104
+ self.matrix: np.ndarray | None = None
105
+ self.rows: int = 0
106
+ self.ready = threading.Event()
107
+ self.error: str | None = None
108
+ self.facets: dict = {}
109
+ self.generation = 0
110
+ self._local = threading.local()
111
+
112
+ def db(self) -> sqlite3.Connection:
113
+ """Thread-local read-only connection, reopened after a swap."""
114
+ conn = getattr(self._local, 'conn', None)
115
+ if conn is not None and \
116
+ getattr(self._local, 'generation', -1) != self.generation:
117
+ # A reload swapped meta.sqlite underneath us; this connection
118
+ # still points at the replaced file.
119
+ conn.close()
120
+ conn = None
121
+ if conn is None:
122
+ conn = sqlite3.connect(
123
+ f'file:{self.spec.local_dir / SQLITE_NAME}?mode=ro', uri=True)
124
+ conn.row_factory = sqlite3.Row
125
+ self._local.conn = conn
126
+ self._local.generation = self.generation
127
+ return conn
128
+
129
+ def invalidate_connections(self) -> None:
130
+ """Bump the generation so every thread reopens on next db()."""
131
+ self.generation += 1
132
+
133
+ def load(self, downloader=None) -> None:
134
+ """Download-if-missing, load, validate, then flip ready.
135
+
136
+ Never raises: failure is recorded in `error` and `ready` stays
137
+ unset so this corpus 503s while the others keep serving.
138
+ Safe to call again after a failure: prior state is dropped up
139
+ front, so a retry can't leave ready/error disagreeing or a stale
140
+ matrix lingering behind the new attempt.
141
+ """
142
+ self.ready.clear()
143
+ self.error = None
144
+ self.matrix = None
145
+ self.rows = 0
146
+ self.facets = {}
147
+ try:
148
+ if not (self.spec.local_dir / MATRIX_NAME).exists():
149
+ if downloader is None:
150
+ from huggingface_hub import snapshot_download
151
+ downloader = snapshot_download
152
+ download_index(downloader, self.spec)
153
+ matrix = np.load(
154
+ self.spec.local_dir / MATRIX_NAME).astype(np.float32)
155
+ _validate_matrix(self.spec, matrix)
156
+ check = sqlite3.connect(
157
+ f'file:{self.spec.local_dir / SQLITE_NAME}?mode=ro', uri=True)
158
+ try:
159
+ _validate_sqlite(self.spec, check, matrix_rows=matrix.shape[0])
160
+ finally:
161
+ check.close()
162
+ self.matrix = matrix
163
+ self.rows = int(matrix.shape[0])
164
+ # A repeated load may be reading replaced files; make every
165
+ # thread reopen on next db() the same way a reload does.
166
+ self.invalidate_connections()
167
+ if self.adapter is not None:
168
+ self.facets = self.adapter.compute_facets(
169
+ self.db(), self.rows)
170
+ self.ready.set()
171
+ except Exception as e: # noqa: BLE001 β€” isolation is the contract
172
+ self.error = f'{type(e).__name__}: {e}'
173
+
174
+ def reload(self, downloader=None) -> bool:
175
+ """Synchronize this corpus from the Hub and hot-swap it in place.
176
+
177
+ Unlike `load`, this never skips the downloader call: the whole point is
178
+ to pick up a freshly published index whose files are already on disk.
179
+ The Hub client still uses its normal metadata diffing, so unchanged
180
+ artifacts are not downloaded again.
181
+
182
+ Sequenced so the slow part happens while the old index is still
183
+ serving β€” download, matrix load and validation all run first, and
184
+ readiness only drops for the couple of seconds it takes to swap
185
+ pointers and recompute facets. That is a short 503 window instead of
186
+ the alternative, which is answering a query against a new matrix and
187
+ a stale meta.sqlite and returning confidently wrong rows.
188
+
189
+ Never raises. On failure the previous matrix, facets and readiness
190
+ are left untouched and `error` records why: a bad publish must not
191
+ take search down. Returns True only when the swap completed.
192
+ """
193
+ try:
194
+ if downloader is None:
195
+ from huggingface_hub import snapshot_download
196
+ downloader = snapshot_download
197
+ download_index(downloader, self.spec)
198
+ matrix = np.load(
199
+ self.spec.local_dir / MATRIX_NAME).astype(np.float32)
200
+ _validate_matrix(self.spec, matrix)
201
+ check = sqlite3.connect(
202
+ f'file:{self.spec.local_dir / SQLITE_NAME}?mode=ro', uri=True)
203
+ try:
204
+ _validate_sqlite(self.spec, check, matrix_rows=matrix.shape[0])
205
+ finally:
206
+ check.close()
207
+ except Exception as e: # noqa: BLE001 β€” keep serving the old index
208
+ self.error = f'reload failed: {type(e).__name__}: {e}'
209
+ return False
210
+
211
+ self.ready.clear()
212
+ try:
213
+ self.matrix = matrix
214
+ self.rows = int(matrix.shape[0])
215
+ # Every thread still holds a connection to the replaced file.
216
+ self.invalidate_connections()
217
+ if self.adapter is not None:
218
+ self.facets = self.adapter.compute_facets(self.db(), self.rows)
219
+ self.error = None
220
+ except Exception as e: # noqa: BLE001 β€” matrix is valid; serve anyway
221
+ self.error = f'reload facets failed: {type(e).__name__}: {e}'
222
+ finally:
223
+ self.ready.set()
224
+ return True
225
+
226
+
227
+ def download_index(downloader, spec: IndexSpec) -> None:
228
+ """Pull the corpus files from the Hub into spec.local_dir."""
229
+ downloader(
230
+ spec.dataset_id,
231
+ repo_type='dataset',
232
+ local_dir=spec.local_dir,
233
+ allow_patterns=list(spec.files),
234
+ )
235
+
236
+
237
+ class CorpusAdapter(Protocol):
238
+ """Schema-specific knowledge for one corpus.
239
+
240
+ The core owns download/load/validate/rank; the adapter owns everything
241
+ that differs between models, kernels, and Docker images.
242
+ """
243
+ spec: IndexSpec
244
+ card_columns: str # SQL select list for result cards
245
+ result_key: str # response key: 'models' | 'kernels' | ...
246
+
247
+ def extract_constraints(self, req) -> str:
248
+ """Move structured constraints out of the request/query and into
249
+ the request's filter slots (never loosening explicit ones).
250
+ Returns the remaining text to embed; '' skips the embedding."""
251
+
252
+ def build_where(self, req) -> tuple[str, list]:
253
+ """SQL WHERE clause + args over this corpus's table."""
254
+
255
+ def compute_facets(self, conn: sqlite3.Connection, total: int) -> dict:
256
+ """Corpus-shaped facet payload served by /api/facets."""
257
+
258
+ def card(self, row: sqlite3.Row, score: float | None = None) -> dict:
259
+ """Serialize one metadata row into a result card."""
260
+
261
+ def sort_order(self, sort: str) -> str:
262
+ """ORDER BY expression for structured (non-semantic) sorts."""
263
+
264
+
265
+ class IndexRegistry:
266
+ """All corpora: registration, isolated loading, per-index status."""
267
+
268
+ def __init__(self, downloader=None) -> None:
269
+ self._indexes: dict[str, LoadedIndex] = {}
270
+ self.downloader = downloader
271
+
272
+ def register(self, adapter) -> LoadedIndex:
273
+ index = LoadedIndex(adapter.spec, adapter)
274
+ self._indexes[adapter.spec.key] = index
275
+ return index
276
+
277
+ def index(self, key: str) -> LoadedIndex:
278
+ return self._indexes[key]
279
+
280
+ def keys(self) -> list[str]:
281
+ return list(self._indexes)
282
+
283
+ def load_all(self) -> None:
284
+ """Load every corpus inline; failures are isolated per index."""
285
+ for index in self._indexes.values():
286
+ index.load(self.downloader)
287
+
288
+ def start_loading(self) -> None:
289
+ """Load every corpus on its own background thread (boot path)."""
290
+ for index in self._indexes.values():
291
+ threading.Thread(
292
+ target=index.load, args=(self.downloader,),
293
+ name=f'bsides-load-{index.spec.key}', daemon=True).start()
294
+
295
+ def status(self) -> dict:
296
+ return {key: {'ready': index.ready.is_set(),
297
+ 'rows': index.rows,
298
+ 'error': index.error}
299
+ for key, index in self._indexes.items()}
300
+
301
+
302
+ def embed_query(index: LoadedIndex, client, q: str, *,
303
+ credential_name: str = 'OPENAI_API_KEY') -> np.ndarray:
304
+ """Embed one query against this index's model and matrix dimensions."""
305
+ if client is None:
306
+ raise UnavailableError(
307
+ f'semantic search unavailable: no {credential_name}')
308
+ if index.matrix is None:
309
+ raise UnavailableError(
310
+ 'semantic search unavailable: index not loaded')
311
+ resp = client.embeddings.create(
312
+ model=index.spec.embed_model,
313
+ dimensions=int(index.matrix.shape[1]),
314
+ input=q[:MAX_QUERY_CHARS],
315
+ )
316
+ v = np.asarray(resp.data[0].embedding, dtype=np.float32)
317
+ return v / max(np.linalg.norm(v), 1e-8)
318
+
319
+
320
+ def semantic_rank(conn, index: LoadedIndex, adapter: CorpusAdapter, *,
321
+ where: str, args: list, sem_q: str, page: int,
322
+ page_size: int, embed_client,
323
+ credential_name: str = 'OPENAI_API_KEY') -> tuple[int, list[dict]]:
324
+ """The relevance pipeline, corpus-agnostic.
325
+
326
+ Candidate row ids from the filtered set, one query embedding, matrix
327
+ scoring, argpartition top-k, then card serialization via the adapter.
328
+ Returns (total, cards).
329
+ """
330
+ table = index.spec.table
331
+ rows = np.fromiter(
332
+ (x[0] for x in conn.execute(
333
+ f'SELECT row FROM {table} WHERE {where}', args)),
334
+ dtype=np.int64)
335
+ total = len(rows)
336
+ if total == 0:
337
+ return 0, []
338
+ qv = embed_query(
339
+ index, embed_client, sem_q, credential_name=credential_name)
340
+ mat = index.matrix
341
+ scores = mat[rows] @ qv if total < index.rows // 2 else (mat @ qv)[rows]
342
+ want = min(total, (page + 1) * page_size)
343
+ top = np.argpartition(-scores, want - 1)[:want]
344
+ top = top[np.argsort(-scores[top])][page * page_size:]
345
+ if len(top) == 0:
346
+ return int(total), []
347
+ sel_rows = rows[top]
348
+ sel_scores = scores[top]
349
+ placeholders = ','.join('?' * len(sel_rows))
350
+ by_row = {row['row']: row for row in conn.execute(
351
+ f'SELECT {adapter.card_columns} FROM {table} '
352
+ f'WHERE row IN ({placeholders})', [int(x) for x in sel_rows])}
353
+ cards = [adapter.card(by_row[int(rw)], sc)
354
+ for rw, sc in zip(sel_rows, sel_scores)]
355
+ return int(total), cards
356
+
357
+
358
+ def structured_page(conn, index: LoadedIndex, adapter: CorpusAdapter, *,
359
+ where: str, args: list, sort: str, page: int,
360
+ page_size: int) -> tuple[int, list[dict]]:
361
+ """Non-semantic path: COUNT + ORDER BY with LIMIT/OFFSET paging."""
362
+ table = index.spec.table
363
+ order = adapter.sort_order(sort)
364
+ total = conn.execute(
365
+ f'SELECT COUNT(*) FROM {table} WHERE {where}', args).fetchone()[0]
366
+ cur = conn.execute(
367
+ f'SELECT {adapter.card_columns} FROM {table} WHERE {where} '
368
+ f'ORDER BY {order} LIMIT ? OFFSET ?',
369
+ args + [page_size, page * page_size])
370
+ cards = [adapter.card(row) for row in cur]
371
+ return int(total), cards
372
+
373
+
374
+
375
+ # ── models corpus ───────────────��─────────────────────────────────────────────
376
+ # Everything below reproduces the historical single-index model behavior
377
+ # verbatim β€” request shape, query-constraint parsing, SQL filtering, facet
378
+ # computation and card serialization.
379
+
380
+ MODEL_CARD_COLUMNS = ('row, model_id, author, month, downloads, likes, tags, '
381
+ 'base_model, relation, license, params, model_type, '
382
+ 'architectures, context_len, num_experts, code_imports, '
383
+ 'gated')
384
+
385
+ MODEL_SORT_ORDERS = {
386
+ 'newest': 'created_at DESC', 'oldest': 'created_at ASC',
387
+ 'downloads': 'downloads DESC', 'likes': 'likes DESC',
388
+ }
389
+
390
+ MODEL_FAMILY_TERMS = {
391
+ 'llama': ('llama',),
392
+ 'qwen': ('qwen',),
393
+ 'gemma': ('gemma',),
394
+ 'mistral': ('mistral', 'mixtral'),
395
+ 'deepseek': ('deepseek',),
396
+ 'phi': ('phi',),
397
+ 'granite': ('granite',),
398
+ }
399
+ MODEL_FAMILY_COLUMNS = (
400
+ 'model_id', 'model_type', 'tags', 'base_model', 'architectures')
401
+ MODEL_QUANTIZATION_MARKERS = (
402
+ 'awq', 'gptq', 'gguf', 'ggml', 'exl2', 'exllama', 'bnb',
403
+ 'bitsandbytes', 'hqq', 'quanto', 'aqlm', 'spqr', 'marlin',
404
+ 'int4', '4bit', '4-bit', '8bit', '8-bit')
405
+
406
+ METHOD_TAGS = ['sft', 'grpo', 'dpo', 'orpo', 'kto', 'ppo', 'lora', 'merge',
407
+ 'unsloth', 'trl', 'axolotl', 'llama-factory', 'autotrain',
408
+ 'custom_code']
409
+
410
+ # methods a user might actually type into the query bar (skip hyphenated/internal ids)
411
+ QUERY_METHODS = ['sft', 'grpo', 'dpo', 'orpo', 'kto', 'ppo', 'lora', 'merge',
412
+ 'unsloth', 'axolotl', 'autotrain']
413
+
414
+ _MONTHS = {m[:3]: i for i, m in enumerate(
415
+ ['jan', 'feb', 'mar', 'apr', 'may', 'jun', 'jul',
416
+ 'aug', 'sep', 'oct', 'nov', 'dec'], 1)}
417
+
418
+ # words that carry no meaning once constraints are stripped β€” drop before embedding
419
+ _STOP = {'a', 'an', 'the', 'from', 'with', 'and', 'or', 'of', 'for', 'in', 'on',
420
+ 'to', 'at', 'by', 'that', 'this', 'is', 'are', 'has', 'have', 'than',
421
+ 'less', 'more', 'under', 'over', 'below', 'above', 'model', 'models',
422
+ 'params', 'param', 'downloads', 'download', 'dl', 'billion', 'b'}
423
+
424
+ ARCH_TAGS = {
425
+ 'llama', 'qwen2', 'qwen3', 'gpt2', 'mistral', 'gemma', 'gemma2', 'gemma3_text',
426
+ 'stablelm', 'gpt_neox', 'phi', 'phi3', 'mixtral', 'falcon', 'mamba', 'mamba2',
427
+ 'jamba', 'rwkv', 'olmo', 'olmo2', 'bloom', 'opt', 'gptj', 'gpt_bigcode',
428
+ 'starcoder2', 'deepseek_v2', 'deepseek_v3', 'cohere', 'dbrx', 'exaone',
429
+ 'granite', 'internlm2', 'minicpm', 'nemotron', 'persimmon', 'plamo',
430
+ 'recurrent_gemma', 'smollm', 'xglm', 'baichuan', 'chatglm', 'yi',
431
+ }
432
+
433
+
434
+ class SearchReq(BaseModel):
435
+ query: str = ''
436
+ filters: dict[str, object] | None = None
437
+ month_from: str | None = None
438
+ month_to: str | None = None
439
+ archs: list[str] = Field(default_factory=list)
440
+ methods: list[str] = Field(default_factory=list)
441
+ relation: str | None = None
442
+ base_model: str | None = None # substring match
443
+ author: str | None = None # substring match
444
+ license: str | None = None
445
+ imports: list[str] = Field(default_factory=list) # code_imports must contain all
446
+ params_min: int | None = None
447
+ params_max: int | None = None
448
+ context_min: int | None = None
449
+ downloads_min: int | None = None
450
+ downloads_max: int | None = None
451
+ moe_only: bool = False
452
+ custom_code_only: bool = False
453
+ config_contains: str | None = None # raw substring over full config json
454
+ exclude_families: list[str] = Field(default_factory=list)
455
+ exclude_quantizations: bool = False
456
+ sort: str = 'relevance' # relevance | newest | oldest | downloads | likes
457
+ page: int = 0
458
+
459
+
460
+
461
+ def parse_query_constraints(q: str) -> tuple[dict, str]:
462
+ """Pull explicit hard constraints (dates, download/param ranges, methods,
463
+ custom-code) out of a natural-language query so they become SQL filters
464
+ instead of being fed to the embedder as vague semantic vibes.
465
+
466
+ Returns (extracted, remainder) where `extracted` holds SearchReq-style
467
+ overrides and `remainder` is the conceptual text to embed. Only the slots
468
+ the caller leaves empty should be filled β€” never loosen an explicit facet.
469
+ """
470
+ ex: dict = {}
471
+ work = q
472
+
473
+ # ── date: "feb 2025" (single month) else bare "2024" (whole year) ──
474
+ m = re.search(r'\b(jan|feb|mar|apr|may|jun|jul|aug|sep|oct|nov|dec)'
475
+ r'[a-z]*\.?\s+(20\d\d)\b', work)
476
+ if m:
477
+ ym = f'{m.group(2)}-{_MONTHS[m.group(1)]:02d}'
478
+ ex['month_from'] = ex['month_to'] = ym
479
+ work = work[:m.start()] + ' ' + work[m.end():]
480
+ else:
481
+ # Require a word boundary so ISO dates like 2025-05 don't have the
482
+ # year extracted from under the reader's feet; a full YYYY-MM string
483
+ # means the caller already knows the month and the parser should not
484
+ # widen it to the whole year.
485
+ m = re.search(r'\b(20\d\d)\b', work)
486
+ if m:
487
+ end = m.end()
488
+ if end < len(work) and work[end] == '-':
489
+ pass
490
+ else:
491
+ ex['month_from'] = f'{m.group(1)}-01'
492
+ ex['month_to'] = f'{m.group(1)}-12'
493
+ work = work[:m.start()] + ' ' + work[m.end():]
494
+
495
+ # ── downloads: "10-50 downloads" range, then < / > comparators (need unit) ──
496
+ m = re.search(r'\b(\d+)\s*(?:-|to|–|β€”)\s*(\d+)\s*downloads?\b', work)
497
+ if m:
498
+ ex['downloads_min'] = int(m.group(1))
499
+ ex['downloads_max'] = int(m.group(2))
500
+ work = work[:m.start()] + ' ' + work[m.end():]
501
+ else:
502
+ m = re.search(r'\b(?:less\s+than|fewer\s+than|under|below|<)\s*(\d+)\s*'
503
+ r'(?:downloads?|dl|↓)\b', work)
504
+ if m:
505
+ ex['downloads_max'] = int(m.group(1)) - 1 # strict: "less than N" β†’ < N
506
+ work = work[:m.start()] + ' ' + work[m.end():]
507
+ else:
508
+ m = re.search(r'\b(?:more\s+than|over|above|>)\s*(\d+)\s*'
509
+ r'(?:downloads?|dl|↓)\b', work)
510
+ if m:
511
+ ex['downloads_min'] = int(m.group(1)) + 1 # strict: "more than N" β†’ > N
512
+ work = work[:m.start()] + ' ' + work[m.end():]
513
+
514
+ # ── params in B: "under 3b", "<3b params", "over 7b" (needs the B suffix) ──
515
+ m = re.search(r'\b(?:under|less\s+than|below|<|≀)\s*(\d+(?:\.\d+)?)\s*'
516
+ r'[bB](?:illion|b)?\s*(?:params?|p)?\b', work)
517
+ if m:
518
+ ex['params_max'] = int(float(m.group(1)) * 1e9)
519
+ work = work[:m.start()] + ' ' + work[m.end():]
520
+ else:
521
+ m = re.search(r'\b(?:over|more\s+than|above|>|β‰₯|at\s+least)\s*(\d+(?:\.\d+)?)\s*'
522
+ r'[bB](?:illion|b)?\s*(?:params?|p)?\b', work)
523
+ if m:
524
+ ex['params_min'] = int(float(m.group(1)) * 1e9)
525
+ work = work[:m.start()] + ' ' + work[m.end():]
526
+
527
+ # ── custom code: "custom code" / "custom-code" ──
528
+ if re.search(r'\bcustom[ -]?code\b', work):
529
+ ex['custom_code_only'] = True
530
+ work = re.sub(r'\bcustom[ -]?code\b', ' ', work)
531
+
532
+ # ── method tags (literal words; left in the embedding β€” they're meaning) ──
533
+ ex['methods'] = [mt for mt in QUERY_METHODS
534
+ if re.search(r'\b' + re.escape(mt) + r'\b', work)]
535
+
536
+ remainder = re.sub(r'\s+', ' ', work).strip()
537
+ meaningful = [w for w in remainder.split() if w.lower() not in _STOP and len(w) > 2]
538
+ # if nothing conceptual survives, return an empty remainder so the caller
539
+ # skips the embedding call and just sorts the filtered set
540
+ return ex, (' '.join(meaningful) if meaningful else '')
541
+
542
+
543
+
544
+ class ModelAdapter:
545
+ """The models corpus: today's single-index behavior, relocated intact."""
546
+
547
+ card_columns = MODEL_CARD_COLUMNS
548
+ result_key = 'models'
549
+
550
+ def __init__(self, spec: IndexSpec) -> None:
551
+ self.spec = spec
552
+
553
+ def extract_constraints(self, r: SearchReq) -> str:
554
+ """Parse the query, fill only the slots left empty, return the
555
+ remaining semantic text ('' skips the embedding call)."""
556
+ ex, sem_q = parse_query_constraints(r.query.strip())
557
+ if ex.get('month_from') and not r.month_from:
558
+ r.month_from = ex['month_from']
559
+ if ex.get('month_to') and not r.month_to:
560
+ r.month_to = ex['month_to']
561
+ if ex.get('downloads_min') is not None and r.downloads_min is None:
562
+ r.downloads_min = ex['downloads_min']
563
+ if ex.get('downloads_max') is not None and r.downloads_max is None:
564
+ r.downloads_max = ex['downloads_max']
565
+ if ex.get('params_min') is not None and r.params_min is None:
566
+ r.params_min = ex['params_min']
567
+ if ex.get('params_max') is not None and r.params_max is None:
568
+ r.params_max = ex['params_max']
569
+ if ex.get('custom_code_only'):
570
+ r.custom_code_only = True
571
+ for mth in ex.get('methods', []):
572
+ if mth not in r.methods:
573
+ r.methods.append(mth)
574
+ return sem_q
575
+
576
+ def build_where(self, r: SearchReq) -> tuple[str, list]:
577
+ where, args = ['1=1'], []
578
+ if r.month_from:
579
+ where.append('month >= ?'); args.append(r.month_from)
580
+ if r.month_to:
581
+ where.append('month <= ?'); args.append(r.month_to)
582
+ if r.archs:
583
+ ors = []
584
+ for a in r.archs:
585
+ ors.append('(model_type = ? OR tags LIKE ?)')
586
+ args += [a, f'%"{a}"%']
587
+ where.append('(' + ' OR '.join(ors) + ')')
588
+ for m in r.methods:
589
+ where.append('tags LIKE ?'); args.append(f'%"{m}"%')
590
+ if r.relation:
591
+ where.append('relation = ?'); args.append(r.relation)
592
+ if r.base_model:
593
+ where.append('base_model LIKE ?'); args.append(f'%{r.base_model}%')
594
+ if r.author:
595
+ where.append('author LIKE ?'); args.append(f'%{r.author}%')
596
+ if r.license:
597
+ where.append('license = ?'); args.append(r.license)
598
+ for imp in r.imports:
599
+ where.append('code_imports LIKE ?'); args.append(f'%"{imp}"%')
600
+ if r.params_min is not None:
601
+ where.append('params >= ?'); args.append(r.params_min)
602
+ if r.params_max is not None:
603
+ where.append('params <= ?'); args.append(r.params_max)
604
+ if r.context_min is not None:
605
+ where.append('context_len >= ?'); args.append(r.context_min)
606
+ if r.downloads_min is not None:
607
+ where.append('downloads >= ?'); args.append(r.downloads_min)
608
+ if r.downloads_max is not None:
609
+ where.append('downloads <= ?'); args.append(r.downloads_max)
610
+ if r.moe_only:
611
+ where.append('num_experts > 1')
612
+ if r.custom_code_only:
613
+ where.append("(code_imports IS NOT NULL OR tags LIKE '%\"custom_code\"%')")
614
+ if r.config_contains:
615
+ where.append('config LIKE ?'); args.append(f'%{r.config_contains}%')
616
+ for family in r.exclude_families:
617
+ terms = MODEL_FAMILY_TERMS.get(family.lower())
618
+ if not terms:
619
+ continue
620
+ family_args = []
621
+ rendered = []
622
+ for column_name in MODEL_FAMILY_COLUMNS:
623
+ for term in terms:
624
+ rendered.append(
625
+ f"LOWER(COALESCE({column_name}, '')) LIKE ?")
626
+ family_args.append(f'%{term}%')
627
+ where.append('NOT (' + ' OR '.join(rendered) + ')')
628
+ args.extend(family_args)
629
+ if r.exclude_quantizations:
630
+ rendered = []
631
+ quant_args = []
632
+ for column_name in MODEL_FAMILY_COLUMNS:
633
+ for marker in MODEL_QUANTIZATION_MARKERS:
634
+ rendered.append(
635
+ f"LOWER(COALESCE({column_name}, '')) LIKE ?")
636
+ quant_args.append(f'%{marker}%')
637
+ rendered.append(
638
+ "CASE WHEN json_valid(config) "
639
+ "THEN json_extract(config, '$.quantization_config') IS NOT NULL "
640
+ "ELSE 0 END")
641
+ where.append('NOT (' + ' OR '.join(rendered) + ')')
642
+ args.extend(quant_args)
643
+ return ' AND '.join(where), args
644
+
645
+ def sort_order(self, sort: str) -> str:
646
+ return MODEL_SORT_ORDERS.get(sort, 'created_at DESC')
647
+
648
+ def card(self, row: sqlite3.Row, score: float | None = None) -> dict:
649
+ d = dict(row)
650
+ d['tags'] = orjson.loads(d['tags']) if d['tags'] else []
651
+ d['code_imports'] = orjson.loads(d['code_imports']) if d['code_imports'] else []
652
+ d['architectures'] = orjson.loads(d['architectures']) if d['architectures'] else []
653
+ if score is not None:
654
+ d['score'] = round(float(score), 4)
655
+ return d
656
+
657
+
658
+ def compute_facets(self, c: sqlite3.Connection, total: int) -> dict:
659
+ months = [r[0] for r in c.execute(
660
+ 'SELECT DISTINCT month FROM models ORDER BY month')]
661
+ licenses = [{'v': r[0], 'n': r[1]} for r in c.execute(
662
+ 'SELECT license, COUNT(*) FROM models WHERE license IS NOT NULL '
663
+ 'GROUP BY license ORDER BY 2 DESC LIMIT 30')]
664
+ relations = [{'v': r[0], 'n': r[1]} for r in c.execute(
665
+ 'SELECT relation, COUNT(*) FROM models WHERE relation IS NOT NULL '
666
+ 'GROUP BY relation ORDER BY 2 DESC')]
667
+ # Prolific authors are a real cluster axis: one person's run of regional
668
+ # language finetunes, one lab's architecture probes.
669
+ authors = [{'v': r[0], 'n': r[1]} for r in c.execute(
670
+ 'SELECT author, COUNT(*) FROM models WHERE author IS NOT NULL '
671
+ 'GROUP BY author ORDER BY 2 DESC LIMIT 40')]
672
+
673
+ enriched = c.execute(
674
+ 'SELECT COUNT(*) FROM models WHERE model_type IS NOT NULL').fetchone()[0]
675
+ if enriched:
676
+ archs = [{'v': r[0], 'n': r[1]} for r in c.execute(
677
+ 'SELECT model_type, COUNT(*) FROM models '
678
+ 'WHERE model_type IS NOT NULL '
679
+ 'GROUP BY model_type ORDER BY 2 DESC LIMIT 40')]
680
+ imports = {}
681
+ for (ci,) in c.execute(
682
+ 'SELECT code_imports FROM models '
683
+ 'WHERE code_imports IS NOT NULL'):
684
+ for mod in orjson.loads(ci):
685
+ imports[mod] = imports.get(mod, 0) + 1
686
+ top_imports = [{'v': k, 'n': v} for k, v in
687
+ sorted(imports.items(), key=lambda x: -x[1])[:40]]
688
+ else:
689
+ # pre-enrichment fallback: architecture tags from the hub tag vocabulary
690
+ arch_counts = {}
691
+ for (tags,) in c.execute('SELECT tags FROM models'):
692
+ for t in orjson.loads(tags):
693
+ if t in ARCH_TAGS:
694
+ arch_counts[t] = arch_counts.get(t, 0) + 1
695
+ archs = [{'v': k, 'n': v} for k, v in
696
+ sorted(arch_counts.items(), key=lambda x: -x[1])[:40]]
697
+ top_imports = []
698
+
699
+ methods = []
700
+ for m in METHOD_TAGS:
701
+ n = c.execute('SELECT COUNT(*) FROM models WHERE tags LIKE ?',
702
+ (f'%"{m}"%',)).fetchone()[0]
703
+ if n:
704
+ methods.append({'v': m, 'n': n})
705
+
706
+ return {
707
+ 'total': total, 'months': months, 'archs': archs,
708
+ 'methods': methods, 'licenses': licenses, 'relations': relations,
709
+ 'imports': top_imports, 'authors': authors,
710
+ 'enriched': bool(enriched),
711
+ }
712
+
713
+
714
+ # ── kernels corpus ────────────────────────────────────────────────────────────
715
+
716
+ KERNEL_CARD_COLUMNS = (
717
+ 'row, repo_id, author, created_at, month, last_modified, downloads, '
718
+ 'likes, tags, languages, accelerators, torch_versions, cuda_archs, '
719
+ 'cpu_archs, operating_systems, dtypes, intrinsics, kernel_names, '
720
+ 'torch_ops, build_variants, variant_count, has_build_dir, source_files'
721
+ )
722
+
723
+ KERNEL_JSON_COLUMNS = (
724
+ 'tags', 'languages', 'accelerators', 'torch_versions', 'cuda_archs',
725
+ 'cpu_archs', 'operating_systems', 'dtypes', 'intrinsics', 'kernel_names',
726
+ 'torch_ops', 'build_variants',
727
+ )
728
+
729
+ KERNEL_ARRAY_FILTERS = (
730
+ 'languages', 'accelerators', 'cuda_archs', 'dtypes', 'intrinsics',
731
+ 'torch_ops', 'torch_versions',
732
+ )
733
+
734
+ KERNEL_SORT_ORDERS = {
735
+ 'newest': 'created_at DESC', 'oldest': 'created_at ASC',
736
+ 'downloads': 'downloads DESC', 'likes': 'likes DESC',
737
+ }
738
+
739
+
740
+ class KernelAdapter:
741
+ """Search behavior for the published kernel-repository index."""
742
+
743
+ card_columns = KERNEL_CARD_COLUMNS
744
+ result_key = 'kernels'
745
+
746
+ def __init__(self, spec: IndexSpec) -> None:
747
+ self.spec = spec
748
+
749
+ def extract_constraints(self, r: SearchReq) -> str:
750
+ # Kernel-specific structure is explicit in `filters`; query prose stays
751
+ # intact for semantic ranking instead of entering the model parser.
752
+ return r.query.strip()
753
+
754
+ def build_where(self, r: SearchReq) -> tuple[str, list]:
755
+ filters = r.filters or {}
756
+ where, args = ['1=1'], []
757
+
758
+ author = filters.get('author')
759
+ if isinstance(author, str) and author:
760
+ where.append('author LIKE ?')
761
+ args.append(f'%{author}%')
762
+
763
+ # month is stored as 'YYYY-MM' text; reject anything that is not
764
+ # zero-padded so a fat-fingered '2024-1' filter can't silently match
765
+ # nothing instead of erroring where the caller can see it.
766
+ for key in ('month_from', 'month_to'):
767
+ value = filters.get(key)
768
+ if value is None:
769
+ continue
770
+ if (isinstance(value, str)
771
+ and re.fullmatch(r'\d{4}-(0[1-9]|1[0-2])', value)):
772
+ where.append(f"month {('>=', '<=')[key == 'month_to']} ?")
773
+ args.append(value)
774
+ else:
775
+ raise UnavailableError(
776
+ f'{key} filter must be a zero-padded YYYY-MM string')
777
+
778
+ # JSON has no int type, so a UI that ships '50' as a string must not
779
+ # end up comparing a TEXT arg against an INTEGER column (which in
780
+ # SQLite never matches and returns an empty page with no error).
781
+ for key, column in (('downloads_min', 'downloads'),
782
+ ('downloads_max', 'downloads'),
783
+ ('variant_count_min', 'variant_count')):
784
+ value = filters.get(key)
785
+ if value is None:
786
+ continue
787
+ operator = '<=' if key.endswith('_max') else '>='
788
+ try:
789
+ coerced = int(value)
790
+ except (TypeError, ValueError) as exc:
791
+ raise UnavailableError(
792
+ f'{key} filter must be an integer') from exc
793
+ where.append(f'{column} {operator} ?')
794
+ args.append(coerced)
795
+
796
+ for column in KERNEL_ARRAY_FILTERS:
797
+ selected = filters.get(column)
798
+ if not isinstance(selected, (list, tuple)):
799
+ continue
800
+ values = [value for value in selected
801
+ if isinstance(value, str) and value]
802
+ if not values:
803
+ continue
804
+ placeholders = ','.join('?' * len(values))
805
+ where.append(
806
+ f'EXISTS (SELECT 1 FROM json_each(kernels.{column}) AS item '
807
+ f'WHERE item.value IN ({placeholders}))')
808
+ args.extend(values)
809
+
810
+ has_build_dir = filters.get('has_build_dir')
811
+ if isinstance(has_build_dir, bool):
812
+ where.append('has_build_dir = ?')
813
+ args.append(int(has_build_dir))
814
+
815
+ return ' AND '.join(where), args
816
+
817
+ def sort_order(self, sort: str) -> str:
818
+ return KERNEL_SORT_ORDERS.get(sort, 'created_at DESC')
819
+
820
+ def card(self, row: sqlite3.Row, score: float | None = None) -> dict:
821
+ card = dict(row)
822
+ for column in KERNEL_JSON_COLUMNS:
823
+ card[column] = orjson.loads(card[column]) if card[column] else []
824
+ card['has_build_dir'] = bool(card['has_build_dir'])
825
+ if score is not None:
826
+ card['score'] = round(float(score), 4)
827
+ return card
828
+
829
+ def compute_facets(self, c: sqlite3.Connection, total: int) -> dict:
830
+ months = [row[0] for row in c.execute(
831
+ 'SELECT DISTINCT month FROM kernels WHERE month IS NOT NULL '
832
+ 'ORDER BY month')]
833
+ authors = [{'v': row[0], 'n': row[1]} for row in c.execute(
834
+ 'SELECT author, COUNT(*) FROM kernels WHERE author IS NOT NULL '
835
+ 'GROUP BY author ORDER BY 2 DESC, 1 ASC LIMIT 40')]
836
+
837
+ facets = {'total': total, 'months': months, 'authors': authors}
838
+ for column in KERNEL_ARRAY_FILTERS:
839
+ counts: dict[str, int] = {}
840
+ for (payload,) in c.execute(
841
+ f'SELECT {column} FROM kernels WHERE {column} IS NOT NULL'):
842
+ for value in orjson.loads(payload):
843
+ counts[value] = counts.get(value, 0) + 1
844
+ facets[column] = [
845
+ {'v': value, 'n': count}
846
+ for value, count in sorted(
847
+ counts.items(), key=lambda item: (-item[1], item[0]))
848
+ ]
849
+
850
+ facets['has_build_dir'] = [
851
+ {'v': bool(value), 'n': count}
852
+ for value, count in c.execute(
853
+ 'SELECT has_build_dir, COUNT(*) FROM kernels '
854
+ 'GROUP BY has_build_dir ORDER BY has_build_dir DESC')
855
+ ]
856
+ return facets
857
+
858
+
859
+ # ── containers corpus ────────────────────────────────────────────────────────
860
+
861
+ CONTAINER_CARD_COLUMNS = (
862
+ 'row, repo_id, repo_type, artifact_path, artifact_kind, author, sha, '
863
+ 'created_at, last_modified, downloads, likes, tags, description, '
864
+ 'artifact_files, qualification_reasons, base_images, accelerators, '
865
+ 'cuda_versions, rocm_versions, python_versions, node_versions, '
866
+ 'operating_systems, ports, services, package_managers, frameworks, '
867
+ 'entrypoints, commands, has_compose, has_devcontainer, has_multistage, '
868
+ 'source_files, source_bytes'
869
+ )
870
+
871
+ CONTAINER_JSON_COLUMNS = (
872
+ 'tags', 'artifact_files', 'qualification_reasons', 'base_images',
873
+ 'accelerators', 'cuda_versions', 'rocm_versions', 'python_versions',
874
+ 'node_versions', 'operating_systems', 'ports', 'services',
875
+ 'package_managers', 'frameworks', 'entrypoints', 'commands',
876
+ )
877
+
878
+ CONTAINER_SCALAR_FILTERS = {
879
+ 'repo_types': 'repo_type',
880
+ 'artifact_kinds': 'artifact_kind',
881
+ }
882
+
883
+ CONTAINER_ARRAY_FILTERS = (
884
+ 'base_images', 'accelerators', 'cuda_versions', 'rocm_versions',
885
+ 'python_versions', 'node_versions', 'operating_systems', 'ports',
886
+ 'services', 'package_managers', 'frameworks',
887
+ )
888
+
889
+ CONTAINER_BOOL_FILTERS = {
890
+ 'has_compose': 'has_compose',
891
+ 'has_devcontainer': 'has_devcontainer',
892
+ 'has_multistage': 'has_multistage',
893
+ }
894
+
895
+ CONTAINER_SORT_ORDERS = {
896
+ 'newest': 'created_at DESC', 'oldest': 'created_at ASC',
897
+ 'downloads': 'downloads DESC', 'likes': 'likes DESC',
898
+ }
899
+
900
+
901
+ class ContainerAdapter:
902
+ """Search behavior for the published container-artifact index."""
903
+
904
+ card_columns = CONTAINER_CARD_COLUMNS
905
+ result_key = 'containers'
906
+
907
+ def __init__(self, spec: IndexSpec) -> None:
908
+ self.spec = spec
909
+
910
+ def extract_constraints(self, r: SearchReq) -> str:
911
+ return r.query.strip()
912
+
913
+ def build_where(self, r: SearchReq) -> tuple[str, list]:
914
+ filters = r.filters or {}
915
+ where, args = ['1=1'], []
916
+
917
+ for key, column in CONTAINER_SCALAR_FILTERS.items():
918
+ selected = filters.get(key)
919
+ if not isinstance(selected, (list, tuple)):
920
+ continue
921
+ values = [value for value in selected
922
+ if isinstance(value, str) and value]
923
+ if not values:
924
+ continue
925
+ placeholders = ','.join('?' * len(values))
926
+ where.append(f'{column} IN ({placeholders})')
927
+ args.extend(values)
928
+
929
+ for column in CONTAINER_ARRAY_FILTERS:
930
+ selected = filters.get(column)
931
+ if not isinstance(selected, (list, tuple)):
932
+ continue
933
+ values = [value for value in selected
934
+ if isinstance(value, (str, int)) and str(value) != '']
935
+ if not values:
936
+ continue
937
+ placeholders = ','.join('?' * len(values))
938
+ # ports are published as JSON integers; comparing the raw value
939
+ # against a TEXT arg via json_each never matches. Normalize both
940
+ # sides to text so '8888' and 8888 hit the same row.
941
+ compare = (f"CAST(item.value AS TEXT) IN ({placeholders})"
942
+ if column == 'ports' else
943
+ f'item.value IN ({placeholders})')
944
+ where.append(
945
+ f'EXISTS (SELECT 1 FROM json_each(containers.{column}) AS item '
946
+ f'WHERE {compare})')
947
+ args.extend([str(v) for v in values] if column == 'ports'
948
+ else values)
949
+
950
+ for key, column in CONTAINER_BOOL_FILTERS.items():
951
+ value = filters.get(key)
952
+ if isinstance(value, bool):
953
+ where.append(f'{column} = ?')
954
+ args.append(int(value))
955
+
956
+ return ' AND '.join(where), args
957
+
958
+ def sort_order(self, sort: str) -> str:
959
+ return CONTAINER_SORT_ORDERS.get(sort, 'created_at DESC')
960
+
961
+ def card(self, row: sqlite3.Row, score: float | None = None) -> dict:
962
+ card = dict(row)
963
+ for column in CONTAINER_JSON_COLUMNS:
964
+ card[column] = orjson.loads(card[column]) if card[column] else []
965
+ for column in CONTAINER_BOOL_FILTERS.values():
966
+ card[column] = bool(card[column])
967
+ if score is not None:
968
+ card['score'] = round(float(score), 4)
969
+ return card
970
+
971
+ def compute_facets(self, c: sqlite3.Connection, total: int) -> dict:
972
+ facets = {'total': total}
973
+
974
+ for key, column in CONTAINER_SCALAR_FILTERS.items():
975
+ facets[key] = [
976
+ {'v': value, 'n': count}
977
+ for value, count in c.execute(
978
+ f'SELECT {column}, COUNT(*) FROM containers '
979
+ f'WHERE {column} IS NOT NULL '
980
+ f'GROUP BY {column} ORDER BY 2 DESC, 1 ASC')
981
+ ]
982
+
983
+ for column in CONTAINER_ARRAY_FILTERS:
984
+ counts: dict[str, int] = {}
985
+ for (payload,) in c.execute(
986
+ f'SELECT {column} FROM containers WHERE {column} IS NOT NULL'):
987
+ for value in orjson.loads(payload):
988
+ counts[value] = counts.get(value, 0) + 1
989
+ facets[column] = [
990
+ {'v': value, 'n': count}
991
+ for value, count in sorted(
992
+ counts.items(), key=lambda item: (-item[1], item[0]))
993
+ ]
994
+
995
+ for key, column in CONTAINER_BOOL_FILTERS.items():
996
+ facets[key] = [
997
+ {'v': bool(value), 'n': count}
998
+ for value, count in c.execute(
999
+ f'SELECT {column}, COUNT(*) FROM containers '
1000
+ f'GROUP BY {column} ORDER BY {column} DESC')
1001
+ ]
1002
+
1003
+ return facets
requirements-dev.txt ADDED
@@ -0,0 +1,5 @@
 
 
 
 
 
 
1
+ # Local tooling only. Deliberately separate from requirements.txt so the Space
2
+ # image does not carry build/export dependencies it never runs.
3
+ -r requirements.txt
4
+ pyarrow==18.1.0
5
+ pytest==9.1.1
requirements.txt ADDED
@@ -0,0 +1,8 @@
 
 
 
 
 
 
 
 
 
1
+ fastapi==0.141.1
2
+ uvicorn[standard]==0.40.0
3
+ numpy==2.4.2
4
+ orjson==3.11.6
5
+ openai==2.16.0
6
+ huggingface_hub==1.26.0
7
+ # not imported directly; pinned because openai and huggingface_hub both need it
8
+ httpx==0.28.1
staff_picks.json ADDED
@@ -0,0 +1,22 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ [
2
+ {
3
+ "model_id": "hoanghai2110/HyperMambaLM-300M",
4
+ "note": "A Mamba state-space backbone with MAML-style meta-learning bolted on β€” built to few-shot adapt. Custom-code-only, ~300M, 11 downloads; the kind of weird SSM hybrid that never makes the charts.",
5
+ "picked_by": "juiceb0xc0de"
6
+ },
7
+ {
8
+ "model_id": "GSAI-ML/ReFusion",
9
+ "note": "A diffusion language model built on Qwen3-8B out of Renmin's Gaoling School β€” it denoises toward text instead of predicting the next token. Custom-code, 8B, and it's a mind fuck to get started.",
10
+ "picked_by": "juiceb0xc0de"
11
+ },
12
+ {
13
+ "model_id": "emozilla/LLongMA-2-7b-storysummarizer",
14
+ "note": "An 8K-context Llama fine-tuned on BookSum to summarize stories β€” a forgotten entry in the LLongMA long-context line. Custom modeling code, 5 downloads, a quiet chapter from the great context-length land grab.",
15
+ "picked_by": "juiceb0xc0de"
16
+ },
17
+ {
18
+ "model_id": "SongTonyLi/OpenELM-270M-CPT-D_chosen-HuggingFaceH4-ultrafeedback_binarized-Xlarge",
19
+ "note": "A 270M OpenELM run through DPO on UltraFeedback β€” the whole preference-tuning pipeline shrunk to pocket size. Custom-code, 2 downloads, and a repo name that won't fit on the label.",
20
+ "picked_by": "juiceb0xc0de"
21
+ }
22
+ ]
static/index.html ADDED
@@ -0,0 +1,1217 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ <!DOCTYPE html>
2
+ <html lang="en">
3
+ <head>
4
+ <meta charset="utf-8">
5
+ <meta name="viewport" content="width=device-width, initial-scale=1">
6
+ <title>B-Sides β€” deep search for indie & vintage LLMs</title>
7
+ <meta name="description" content="Search 228,000+ indie and vintage text-generation models on Hugging Face by meaning, time period, architecture, training method, lineage, config internals like flash_attn β€” the deep cuts HF search can't reach.">
8
+ <link rel="preconnect" href="https://api.fontshare.com">
9
+ <link href="https://api.fontshare.com/v2/css?f[]=gambarino@400&display=swap" rel="stylesheet">
10
+ <link rel="preconnect" href="https://fonts.googleapis.com">
11
+ <link rel="preconnect" href="https://fonts.gstatic.com" crossorigin>
12
+ <link href="https://fonts.googleapis.com/css2?family=IBM+Plex+Mono:ital,wght@0,400;0,500;0,600;1,400&display=swap" rel="stylesheet">
13
+ <script src="https://cdn.jsdelivr.net/npm/gsap@3.12.5/dist/gsap.min.js"></script>
14
+ <script src="https://cdn.jsdelivr.net/npm/vanilla-tilt@1.8.1/dist/vanilla-tilt.min.js"></script>
15
+ <style>
16
+ :root {
17
+ /* tarot: cream parchment, sepia ink, flat pastel fills */
18
+ --paper: #ece2c8;
19
+ --card: #f6efdc;
20
+ --card2: #f0e7cf;
21
+ --ink: #3a2f22;
22
+ --ink-soft: #675941;
23
+ --ink-faint: #93835f;
24
+ --line: #d5c7a3;
25
+ --frame: #8d7a55;
26
+ --rose: #e9bcc4; --rose-deep: #b25a6e;
27
+ --sage: #c8dcba; --sage-deep: #5f8248;
28
+ --powder: #c2dbe6; --powder-deep: #4f7f96;
29
+ --marigold: #eed093; --marigold-deep: #a1782c;
30
+ --terra: #e7ab95; --terra-deep: #ad4f36;
31
+ --lilac: #d7c7e7; --lilac-deep: #7d5fa0;
32
+ --spring: cubic-bezier(.34,1.56,.64,1);
33
+ --ease: cubic-bezier(.22,.61,.36,1);
34
+ --serif: 'Gambarino', 'Georgia', serif;
35
+ --mono: 'IBM Plex Mono', monospace;
36
+ }
37
+ * { box-sizing: border-box; margin: 0; }
38
+ html { scroll-behavior: smooth; }
39
+ body { background: var(--paper); color: var(--ink); font-family: var(--mono); font-size: 14px; line-height: 1.6; min-height: 100vh; overflow-x: hidden; }
40
+ body::after { /* paper grain */
41
+ content: ''; position: fixed; inset: 0; pointer-events: none; z-index: 1;
42
+ opacity: .05; mix-blend-mode: multiply;
43
+ background-image: url("data:image/svg+xml,%3Csvg xmlns='http://www.w3.org/2000/svg' width='180' height='180'%3E%3Cfilter id='n'%3E%3CfeTurbulence type='fractalNoise' baseFrequency='0.85' numOctaves='3' stitchTiles='stitch'/%3E%3C/filter%3E%3Crect width='180' height='180' filter='url(%23n)'/%3E%3C/svg%3E");
44
+ }
45
+ a { color: var(--terra-deep); text-decoration: none; }
46
+ a:hover { text-decoration: underline; }
47
+ ::selection { background: var(--marigold); color: var(--ink); }
48
+ ::placeholder { color: var(--ink-faint); }
49
+ input, select, button { font-family: inherit; font-size: inherit; color: inherit; }
50
+
51
+ .wrap { position: relative; z-index: 2; max-width: 1340px; margin: 0 auto; padding: 0 26px; }
52
+
53
+ /* ── header ── */
54
+ header { display: flex; align-items: center; justify-content: space-between; padding: 16px 0; border-bottom: 1px solid var(--line); }
55
+ .logo { display: flex; align-items: center; gap: 12px; }
56
+ .logo-disc { width: 32px; height: 32px; border-radius: 50%; position: relative; flex: none;
57
+ background: var(--card); border: 1.5px solid var(--ink); }
58
+ .logo-disc::before { content: ''; position: absolute; inset: 4px; border-radius: 50%; border: 1px solid var(--frame); }
59
+ .logo-disc::after { content: ''; position: absolute; inset: 10px; border-radius: 50%; background: var(--terra); border: 1.5px solid var(--ink); }
60
+ .logo h1 { font-family: var(--serif); font-weight: 400; font-size: 25px; letter-spacing: .5px; }
61
+ .tagline { font-size: 10px; letter-spacing: 2.5px; text-transform: uppercase; color: var(--ink-faint); }
62
+ .head-right { font-size: 11px; letter-spacing: 1px; color: var(--ink-soft); display: flex; gap: 14px; align-items: center; }
63
+ .head-right .dot { color: var(--line); }
64
+
65
+ /* ── hero: search front and center ── */
66
+ .hero { display: flex; flex-direction: column; justify-content: center; align-items: center; text-align: center; padding: 34px 0 40px; }
67
+ .hero h2 { font-family: var(--serif); font-weight: 400; font-size: clamp(26px, 3.4vw, 42px); line-height: 1.12; letter-spacing: .2px; max-width: 720px; }
68
+ .hero h2 em { font-style: italic; color: var(--terra-deep); }
69
+ .hero .rule { width: 130px; height: 10px; margin: 14px auto 22px; display: flex; gap: 5px; justify-content: center; }
70
+ .hero .rule i { flex: 1; border-radius: 3px; }
71
+ .searchbar { width: min(720px, 100%); display: flex; align-items: center; gap: 12px;
72
+ background: var(--card); border: 1.5px solid var(--ink); border-radius: 12px;
73
+ padding: 15px 18px; position: relative;
74
+ box-shadow: 4px 4px 0 rgba(58,47,34,.16);
75
+ transition: box-shadow .22s var(--ease), transform .22s var(--ease); }
76
+ .searchbar:focus-within { box-shadow: 6px 7px 0 rgba(58,47,34,.20); transform: translate(-1px,-1px); }
77
+ .searchbar .glyph { color: var(--terra-deep); font-size: 15px; }
78
+ .searchbar input { flex: 1; background: transparent; border: none; outline: none; font-size: 15px; color: var(--ink); min-width: 40px; }
79
+ .corpus-select { flex: none; background: var(--card2); border: 1px solid var(--frame); border-radius: 8px;
80
+ padding: 7px 28px 7px 10px; outline: none; font-size: 11px; font-weight: 600;
81
+ letter-spacing: 1.5px; text-transform: uppercase; cursor: pointer; }
82
+ .corpus-select:focus { border-color: var(--ink); }
83
+ @media (max-width: 620px) {
84
+ .searchbar { gap: 8px; padding: 12px; flex-wrap: wrap; }
85
+ .searchbar .glyph { display: none; }
86
+ .corpus-select { order: 1; }
87
+ .searchbar input { order: 2; width: calc(100% - 118px); }
88
+ .btn-dig { order: 3; margin-left: auto; }
89
+ }
90
+ .btn-dig { border: 1.5px solid var(--ink); cursor: pointer; white-space: nowrap;
91
+ font-size: 11px; font-weight: 600; letter-spacing: 2px; text-transform: uppercase;
92
+ color: var(--ink); background: var(--marigold); border-radius: 9px; padding: 9px 18px;
93
+ box-shadow: 3px 3px 0 var(--ink);
94
+ transition: transform .15s var(--spring), box-shadow .15s var(--spring); }
95
+ .btn-dig:hover { transform: translate(-1px,-2px); box-shadow: 4px 5px 0 var(--ink); }
96
+ .btn-dig:active { transform: translate(3px,3px); box-shadow: 0 0 0 var(--ink); }
97
+ .status-line { margin-top: 14px; font-size: 12.5px; color: var(--ink-soft); min-height: 20px; }
98
+ .status-line b { color: var(--terra-deep); font-weight: 600; }
99
+ .hero .promise { margin-top: 6px; font-size: 11.5px; color: var(--ink-faint); letter-spacing: .4px; max-width: 640px; }
100
+ .hero .promise b { color: var(--ink-soft); font-weight: 500; }
101
+ .hero .manifesto { margin-top: 18px; max-width: 620px; font-family: var(--serif); font-size: 14px; line-height: 1.62; color: var(--ink-soft); }
102
+
103
+ /* ── layout ── */
104
+ .main { display: grid; grid-template-columns: 262px 1fr; gap: 30px; padding: 22px 0 80px; }
105
+ @media (max-width: 900px) { .main { grid-template-columns: 1fr; } }
106
+
107
+ /* rail */
108
+ .rail { position: sticky; top: 14px; align-self: start; display: flex; flex-direction: column; gap: 14px; max-height: calc(100vh - 28px); overflow-y: auto; scrollbar-width: thin; padding: 4px 8px 24px 4px; }
109
+ @media (max-width: 900px) { .rail { position: static; max-height: none; } }
110
+ .filter-stack { display: flex; flex-direction: column; gap: 14px; }
111
+ .knob-group { border: 1.5px solid var(--ink); border-radius: 11px; background: var(--card); padding: 13px 14px; box-shadow: 3px 3px 0 rgba(58,47,34,.12); }
112
+ .knob-group h4 { font-size: 10px; letter-spacing: 2.5px; text-transform: uppercase; margin-bottom: 10px; font-weight: 600; color: var(--ink); display: flex; align-items: center; gap: 8px; }
113
+ .knob-group h4 .swatch { width: 22px; height: 8px; border-radius: 3px; border: 1px solid var(--ink); flex: none; }
114
+ .knob-group h4 .soon { margin-left: auto; color: var(--ink-faint); border: 1px dashed var(--frame); border-radius: 6px; padding: 1px 7px; letter-spacing: 1px; font-size: 9px; text-transform: none; }
115
+ .chips { display: flex; flex-wrap: wrap; gap: 6px; }
116
+ .chip { cursor: pointer; user-select: none; font-size: 11.5px; padding: 3.5px 10px; border-radius: 999px;
117
+ border: 1px solid var(--frame); color: var(--ink-soft); background: transparent;
118
+ transition: transform .16s var(--spring), background .16s, color .16s, border-color .16s; }
119
+ .chip:hover { border-color: var(--ink); color: var(--ink); transform: translateY(-1px); }
120
+ .chip .n { opacity: .55; font-size: 10px; margin-left: 4px; }
121
+ .chip.more-toggle { border-style: dashed; }
122
+ .chip.on { border-color: var(--ink); color: var(--ink); font-weight: 600; box-shadow: 2px 2px 0 rgba(58,47,34,.25); }
123
+ .kg-arch .chip.on { background: var(--marigold); }
124
+ .kg-method .chip.on { background: var(--sage); }
125
+ .kg-deep .chip.on { background: var(--lilac); }
126
+ .kg-kernel .chip.on { background: var(--chip-fill, var(--powder)); }
127
+ .rail select, .rail input[type=text], .rail input[type=number] {
128
+ width: 100%; background: #fbf6e8; border: 1px solid var(--frame); border-radius: 8px;
129
+ padding: 7px 10px; outline: none; font-size: 12.5px; color: var(--ink); transition: border-color .2s; }
130
+ .rail select:focus, .rail input:focus { border-color: var(--ink); }
131
+ .range-row { display: flex; gap: 8px; align-items: center; }
132
+ .range-row span { color: var(--ink-faint); font-size: 11px; }
133
+ .toggle-row { display: flex; align-items: center; justify-content: space-between; margin-top: 9px; font-size: 12.5px; color: var(--ink-soft); cursor: pointer; user-select: none; }
134
+ .switch { width: 34px; height: 19px; border-radius: 999px; background: #fbf6e8; border: 1px solid var(--frame); position: relative; transition: background .2s; flex: none; }
135
+ .switch::after { content: ''; position: absolute; top: 2px; left: 2px; width: 13px; height: 13px; border-radius: 50%; background: var(--ink-faint); transition: all .2s var(--spring); }
136
+ .toggle-row.on .switch { background: var(--lilac); border-color: var(--ink); }
137
+ .toggle-row.on .switch::after { left: 17px; background: var(--ink); }
138
+ .sub-label { font-size: 10px; letter-spacing: 1.6px; text-transform: uppercase; color: var(--ink-soft); margin-bottom: 7px; }
139
+ .sub-label .hint { text-transform: none; letter-spacing: .3px; color: var(--ink-faint); }
140
+ .warn { margin-top: 7px; font-size: 11px; color: var(--terra-deep); }
141
+ .excludes { display: flex; flex-direction: column; gap: 3px; }
142
+ .ex-row { display: flex; align-items: center; gap: 8px; font-size: 12.5px; color: var(--ink-soft); cursor: pointer; user-select: none; }
143
+ .ex-row .box { width: 14px; height: 14px; flex: none; border-radius: 4px; border: 1px solid var(--frame); background: #fbf6e8; position: relative; transition: background .16s, border-color .16s; }
144
+ .ex-row.on { color: var(--ink); }
145
+ .ex-row.on .box { background: var(--terra); border-color: var(--ink); }
146
+ .ex-row.on .box::after { content: 'βœ•'; position: absolute; inset: 0; font-size: 10px; line-height: 13px; text-align: center; color: var(--ink); }
147
+ .exclude-note { margin-top: 9px; font-size: 10.5px; line-height: 1.5; color: var(--ink-faint); }
148
+ .tcard.docker-card .mid { word-break: break-word; font-size: 16px; }
149
+ .tcard.docker-card .pills { max-height: 54px; overflow: hidden; }
150
+ .tcard.docker-card .facts { margin-top: 9px; font-size: 10px; color: var(--ink-faint); letter-spacing: .5px; word-break: break-all; }
151
+ .btn-clear { width: 100%; cursor: pointer; background: transparent; border: 1px dashed var(--frame); border-radius: 10px; padding: 8px; color: var(--ink-faint); font-size: 11px; letter-spacing: 2px; text-transform: uppercase; transition: all .2s; }
152
+ .btn-clear:hover { border-color: var(--terra-deep); color: var(--terra-deep); }
153
+
154
+ /* section heads */
155
+ .crates-head { display: flex; align-items: baseline; justify-content: space-between; margin: 6px 0 16px; flex-wrap: wrap; gap: 10px; }
156
+ .crates-head h3 { font-family: var(--serif); font-weight: 400; font-size: 26px; }
157
+ .crates-head .sub { font-size: 11px; color: var(--ink-faint); letter-spacing: 1px; }
158
+ .sort-row { display: flex; gap: 8px; align-items: center; font-size: 11.5px; color: var(--ink-soft); }
159
+ .sort-row select { background: #fbf6e8; border: 1px solid var(--frame); border-radius: 8px; padding: 5px 9px; font-size: 11.5px; color: var(--ink); outline: none; }
160
+
161
+ /* ── the spread (staff picks) ── */
162
+ #spread { margin-bottom: 34px; }
163
+ .spread-row { display: grid; grid-template-columns: repeat(auto-fill, minmax(230px, 1fr)); gap: 20px; }
164
+ .pick { text-align: center; }
165
+ .pick .note { margin-top: 10px; font-family: var(--serif); font-style: italic; font-size: 13.5px; color: var(--ink-soft); line-height: 1.5; padding: 0 6px; }
166
+ .pick .by { margin-top: 4px; font-size: 10px; letter-spacing: 1.5px; text-transform: uppercase; color: var(--ink-faint); }
167
+
168
+ /* ── tarot card ── */
169
+ .grid { display: grid; grid-template-columns: repeat(auto-fill, minmax(240px, 1fr)); gap: 20px; }
170
+ .tcard { position: relative; cursor: pointer; border-radius: 12px; overflow: hidden;
171
+ background: var(--card); border: 1.5px solid var(--ink);
172
+ box-shadow: 3px 4px 0 rgba(58,47,34,.14);
173
+ transition: transform .22s var(--spring), box-shadow .25s var(--ease);
174
+ transform-style: preserve-3d; }
175
+ .tcard::after { /* inner frame */
176
+ content: ''; position: absolute; inset: 5px; border: 1px solid var(--frame); border-radius: 8px; pointer-events: none; }
177
+ .tcard:hover { transform: translateY(-4px) rotate(-.5deg); box-shadow: 5px 9px 0 rgba(58,47,34,.18); }
178
+ .tcard .sky { /* pastel top band */
179
+ background: var(--tc, var(--powder)); border-bottom: 1px solid var(--ink);
180
+ padding: 7px 14px 6px; display: flex; justify-content: space-between; align-items: center;
181
+ font-size: 10px; letter-spacing: 2px; color: var(--ink); }
182
+ .tcard .sky .num { font-weight: 600; }
183
+ .tcard .body { padding: 14px 15px 12px; text-align: center; }
184
+ .tcard .mid { font-family: var(--serif); font-size: 17.5px; line-height: 1.25; word-break: break-all; }
185
+ .tcard .author { font-size: 10.5px; color: var(--ink-faint); letter-spacing: 1px; margin-top: 3px; }
186
+ .tcard .meta { display: flex; gap: 9px; justify-content: center; margin-top: 9px; font-size: 10.5px; color: var(--ink-soft); flex-wrap: wrap; }
187
+ .tcard .meta .sc-match { color: var(--terra-deep); font-weight: 600; }
188
+ .tcard .pills { display: flex; flex-wrap: wrap; gap: 5px; justify-content: center; margin-top: 9px; min-height: 0; }
189
+ .tcard .pill { font-size: 9.5px; padding: 1.5px 8px; border-radius: 999px; border: 1px solid var(--ink); background: var(--pc, var(--sage)); color: var(--ink); }
190
+ .tcard .pill.hot { background: var(--terra); font-weight: 600; }
191
+ .tcard .lineage { margin-top: 8px; font-size: 10px; color: var(--ink-faint); font-style: italic; overflow: hidden; text-overflow: ellipsis; white-space: nowrap; }
192
+ .tcard .caption { border-top: 1px solid var(--ink); margin-top: 11px; padding: 7px 10px 8px;
193
+ font-size: 10px; letter-spacing: 2.5px; text-transform: uppercase; color: var(--ink);
194
+ background: var(--card2); position: relative; z-index: 0; }
195
+ .tcard.kernel-card .mid { word-break: break-word; }
196
+ .tcard.kernel-card .pills { max-height: 54px; overflow: hidden; }
197
+ .tcard.kernel-card .facts { margin-top: 9px; font-size: 10px; color: var(--ink-faint); letter-spacing: .5px; }
198
+
199
+ /* loader: ink-drawn spinning card */
200
+ .loader { display: none; margin: 46px auto; width: 46px; height: 66px; border-radius: 7px;
201
+ background: var(--card); border: 1.5px solid var(--ink); position: relative;
202
+ animation: flip 1s var(--ease) infinite; }
203
+ .loader::after { content: ''; position: absolute; inset: 5px; border: 1px solid var(--frame); border-radius: 4px; }
204
+ @keyframes flip { 0%{transform:rotateY(0)} 50%{transform:rotateY(180deg)} 100%{transform:rotateY(360deg)} }
205
+ .loading .loader { display: block; }
206
+ .loading .grid, .loading .spread-row { opacity: .35; }
207
+
208
+ .empty { text-align: center; padding: 60px 0; display: none; }
209
+ .empty .kao { font-size: 26px; color: var(--terra-deep); letter-spacing: 2px; }
210
+ .empty h4 { font-family: var(--serif); font-size: 27px; margin-top: 12px; }
211
+ .empty p { color: var(--ink-faint); margin-top: 8px; font-size: 13px; }
212
+
213
+ .btn-more { display: block; margin: 30px auto 0; cursor: pointer;
214
+ background: var(--card); border: 1.5px solid var(--ink); border-radius: 999px;
215
+ color: var(--ink); padding: 10px 30px; font-size: 11px; letter-spacing: 2.5px; text-transform: uppercase;
216
+ box-shadow: 3px 3px 0 rgba(58,47,34,.18);
217
+ transition: transform .18s var(--spring), box-shadow .18s var(--spring); }
218
+ .btn-more:hover { transform: translate(-1px,-2px); box-shadow: 4px 5px 0 rgba(58,47,34,.22); }
219
+ .btn-more:active { transform: translate(3px,3px); box-shadow: 0 0 0 rgba(58,47,34,.2); }
220
+
221
+ footer { border-top: 1px solid var(--line); padding: 22px 0 52px;
222
+ display: flex; justify-content: space-between; flex-wrap: wrap; gap: 12px;
223
+ font-size: 11.5px; color: var(--ink-faint); letter-spacing: .8px; }
224
+ footer .next { color: var(--marigold-deep); }
225
+ </style>
226
+ </head>
227
+ <body>
228
+ <div class="wrap">
229
+
230
+ <header id="hdr">
231
+ <div class="logo">
232
+ <div class="logo-disc"></div>
233
+ <div>
234
+ <h1>B-Sides.</h1>
235
+ <div class="tagline" id="tagline">deep search Β· indie & vintage LLMs</div>
236
+ </div>
237
+ </div>
238
+ <div class="head-right">
239
+ <span>πŸ€— hugging face space</span><span class="dot">|</span><span id="head-count">…</span><span class="dot">|</span><span id="catalog-copy">2022 pressing</span>
240
+ </div>
241
+ </header>
242
+
243
+ <div class="main">
244
+ <aside class="rail" id="rail">
245
+
246
+ <div class="filter-stack" id="model-filters">
247
+
248
+ <div class="knob-group kg-exclude" id="exclude-group">
249
+ <h4><span class="swatch" style="background:var(--terra)"></span>Hidden by default</h4>
250
+ <div class="excludes" id="f-excludes"></div>
251
+ <p class="exclude-note">checked is hidden Β· uncheck to let them back into the crates</p>
252
+ </div>
253
+
254
+ <div class="knob-group kg-time">
255
+ <h4><span class="swatch" style="background:var(--rose)"></span>Time period</h4>
256
+ <div class="range-row">
257
+ <select id="f-month-from"></select><span>β†’</span><select id="f-month-to"></select>
258
+ </div>
259
+ </div>
260
+
261
+ <div class="knob-group kg-arch">
262
+ <h4><span class="swatch" style="background:var(--marigold)"></span>Architecture</h4>
263
+ <div class="chips" id="f-archs"></div>
264
+ </div>
265
+
266
+ <div class="knob-group kg-method">
267
+ <h4><span class="swatch" style="background:var(--sage)"></span>Training method</h4>
268
+ <div class="chips" id="f-methods"></div>
269
+ </div>
270
+
271
+ <div class="knob-group kg-lineage">
272
+ <h4><span class="swatch" style="background:var(--powder)"></span>Lineage</h4>
273
+ <select id="f-relation"><option value="">any relation</option></select>
274
+ <div style="height:8px"></div>
275
+ <input id="f-base" type="text" placeholder="base model contains…">
276
+ </div>
277
+
278
+ <div class="knob-group kg-license">
279
+ <h4><span class="swatch" style="background:var(--terra)"></span>License</h4>
280
+ <select id="f-license"><option value="">any license</option></select>
281
+ </div>
282
+
283
+ <div class="knob-group kg-deep" id="deep-group">
284
+ <h4><span class="swatch" style="background:var(--lilac)"></span>Deep cuts <span class="soon" id="deep-soon" style="display:none">pressing…</span></h4>
285
+ <div id="deep-filters">
286
+ <div class="sub-label">Model size (billions) <span class="hint">e.g. 1.5B–3.5B</span></div>
287
+ <div class="range-row">
288
+ <input id="f-pmin" type="number" placeholder="params β‰₯ (B)" step="0.1" min="0">
289
+ <input id="f-pmax" type="number" placeholder="≀ (B)" step="0.1" min="0">
290
+ </div>
291
+ <div class="warn" id="size-warn" style="display:none">minimum is above maximum</div>
292
+ <div style="height:8px"></div>
293
+ <input id="f-ctx" type="number" placeholder="context β‰₯ tokens">
294
+ <div style="height:8px"></div>
295
+ <input id="f-config" type="text" placeholder='config contains… "rope_scaling"'>
296
+ <div style="height:10px"></div>
297
+ <div class="chips" id="f-imports"></div>
298
+ </div>
299
+ <div class="toggle-row" id="f-moe"><span>MoE only</span><span class="switch"></span></div>
300
+ <div class="toggle-row" id="f-custom"><span>custom code only</span><span class="switch"></span></div>
301
+ </div>
302
+
303
+ </div>
304
+
305
+ <div class="filter-stack" id="kernel-filters" style="display:none">
306
+ <div class="knob-group kg-time">
307
+ <h4><span class="swatch" style="background:var(--rose)"></span>Time period</h4>
308
+ <div class="range-row">
309
+ <select id="k-month-from"></select><span>β†’</span><select id="k-month-to"></select>
310
+ </div>
311
+ </div>
312
+
313
+ <div class="knob-group kg-kernel" style="--chip-fill:var(--rose)">
314
+ <h4><span class="swatch" style="background:var(--rose)"></span>Downloads</h4>
315
+ <div class="range-row">
316
+ <input id="k-downloads-min" type="number" placeholder="downloads β‰₯" min="0">
317
+ <input id="k-downloads-max" type="number" placeholder="≀" min="0">
318
+ </div>
319
+ </div>
320
+
321
+ <div class="knob-group kg-kernel" style="--chip-fill:var(--marigold)">
322
+ <h4><span class="swatch" style="background:var(--marigold)"></span>Languages</h4>
323
+ <div class="chips" id="k-languages"></div>
324
+ </div>
325
+
326
+ <div class="knob-group kg-kernel" style="--chip-fill:var(--sage)">
327
+ <h4><span class="swatch" style="background:var(--sage)"></span>Accelerators</h4>
328
+ <div class="chips" id="k-accelerators"></div>
329
+ </div>
330
+
331
+ <div class="knob-group kg-kernel" style="--chip-fill:var(--powder)">
332
+ <h4><span class="swatch" style="background:var(--powder)"></span>CUDA architectures</h4>
333
+ <div class="chips" id="k-cuda-archs"></div>
334
+ </div>
335
+
336
+ <div class="knob-group kg-kernel" style="--chip-fill:var(--lilac)">
337
+ <h4><span class="swatch" style="background:var(--lilac)"></span>Dtypes</h4>
338
+ <div class="chips" id="k-dtypes"></div>
339
+ </div>
340
+
341
+ <div class="knob-group kg-kernel" style="--chip-fill:var(--terra)">
342
+ <h4><span class="swatch" style="background:var(--terra)"></span>Intrinsics</h4>
343
+ <div class="chips" id="k-intrinsics"></div>
344
+ </div>
345
+
346
+ <div class="knob-group kg-kernel" style="--chip-fill:var(--marigold)">
347
+ <h4><span class="swatch" style="background:var(--marigold)"></span>Torch operations</h4>
348
+ <div class="chips" id="k-torch-ops"></div>
349
+ </div>
350
+
351
+ <div class="knob-group kg-kernel" style="--chip-fill:var(--sage)">
352
+ <h4><span class="swatch" style="background:var(--sage)"></span>Torch versions</h4>
353
+ <div class="chips" id="k-torch-versions"></div>
354
+ </div>
355
+
356
+ <div class="knob-group kg-kernel">
357
+ <h4><span class="swatch" style="background:var(--powder)"></span>Author</h4>
358
+ <select id="k-author"><option value="">any author</option></select>
359
+ </div>
360
+
361
+ <div class="knob-group kg-kernel">
362
+ <h4><span class="swatch" style="background:var(--lilac)"></span>Minimum variants</h4>
363
+ <input id="k-variant-min" type="number" placeholder="variants β‰₯" min="0">
364
+ </div>
365
+
366
+ <div class="knob-group kg-kernel">
367
+ <h4><span class="swatch" style="background:var(--terra)"></span>Build facts</h4>
368
+ <div class="toggle-row" id="k-build"><span>has build directory</span><span class="switch"></span></div>
369
+ </div>
370
+ </div>
371
+
372
+ <div class="filter-stack" id="docker-filters" style="display:none">
373
+ <div class="knob-group kg-kernel" style="--chip-fill:var(--terra)">
374
+ <h4><span class="swatch" style="background:var(--terra)"></span>Artifact kind</h4>
375
+ <div class="toggle-row" id="d-archives"><span>include archives</span><span class="switch"></span></div>
376
+ <p class="exclude-note">recipes only by default Β· archives are tarballs, not build files</p>
377
+ </div>
378
+
379
+ <div class="knob-group kg-kernel" style="--chip-fill:var(--rose)">
380
+ <h4><span class="swatch" style="background:var(--rose)"></span>Repository type</h4>
381
+ <div class="chips" id="d-repo-types"></div>
382
+ </div>
383
+
384
+ <div class="knob-group kg-kernel" style="--chip-fill:var(--sage)">
385
+ <h4><span class="swatch" style="background:var(--sage)"></span>Accelerators</h4>
386
+ <div class="chips" id="d-accelerators"></div>
387
+ </div>
388
+
389
+ <div class="knob-group kg-kernel" style="--chip-fill:var(--powder)">
390
+ <h4><span class="swatch" style="background:var(--powder)"></span>CUDA versions</h4>
391
+ <div class="chips" id="d-cuda-versions"></div>
392
+ </div>
393
+
394
+ <div class="knob-group kg-kernel" style="--chip-fill:var(--terra)">
395
+ <h4><span class="swatch" style="background:var(--terra)"></span>ROCm versions</h4>
396
+ <div class="chips" id="d-rocm-versions"></div>
397
+ </div>
398
+
399
+ <div class="knob-group kg-kernel" style="--chip-fill:var(--marigold)">
400
+ <h4><span class="swatch" style="background:var(--marigold)"></span>Python versions</h4>
401
+ <div class="chips" id="d-python-versions"></div>
402
+ </div>
403
+
404
+ <div class="knob-group kg-kernel" style="--chip-fill:var(--lilac)">
405
+ <h4><span class="swatch" style="background:var(--lilac)"></span>Frameworks</h4>
406
+ <div class="chips" id="d-frameworks"></div>
407
+ </div>
408
+
409
+ <div class="knob-group kg-kernel" style="--chip-fill:var(--sage)">
410
+ <h4><span class="swatch" style="background:var(--sage)"></span>Base images</h4>
411
+ <div class="chips" id="d-base-images"></div>
412
+ </div>
413
+
414
+ <div class="knob-group kg-kernel" style="--chip-fill:var(--powder)">
415
+ <h4><span class="swatch" style="background:var(--powder)"></span>Operating systems</h4>
416
+ <div class="chips" id="d-operating-systems"></div>
417
+ </div>
418
+
419
+ <div class="knob-group kg-kernel" style="--chip-fill:var(--marigold)">
420
+ <h4><span class="swatch" style="background:var(--marigold)"></span>Package managers</h4>
421
+ <div class="chips" id="d-package-managers"></div>
422
+ </div>
423
+
424
+ <div class="knob-group kg-kernel" style="--chip-fill:var(--rose)">
425
+ <h4><span class="swatch" style="background:var(--rose)"></span>Services</h4>
426
+ <div class="chips" id="d-services"></div>
427
+ </div>
428
+
429
+ <div class="knob-group kg-kernel" style="--chip-fill:var(--lilac)">
430
+ <h4><span class="swatch" style="background:var(--lilac)"></span>Node versions</h4>
431
+ <div class="chips" id="d-node-versions"></div>
432
+ </div>
433
+
434
+ <!-- No exposed-ports control: ports are published as JSON integers and
435
+ the adapter only accepts string filter values, so any chip here
436
+ would match nothing. Needs a backend cast before it ships. -->
437
+
438
+ <div class="knob-group kg-kernel">
439
+ <h4><span class="swatch" style="background:var(--powder)"></span>Build facts</h4>
440
+ <div class="toggle-row" id="d-compose"><span>has compose file</span><span class="switch"></span></div>
441
+ <div class="toggle-row" id="d-devcontainer"><span>has devcontainer</span><span class="switch"></span></div>
442
+ <div class="toggle-row" id="d-multistage"><span>multi-stage build</span><span class="switch"></span></div>
443
+ </div>
444
+ </div>
445
+
446
+ <button class="btn-clear" id="clear">βœ• clear the deck</button>
447
+ </aside>
448
+
449
+ <section>
450
+ <section class="hero">
451
+ <h2 id="hero-h">The models that never charted.<br><em>Dig anyway.</em></h2>
452
+ <div class="rule" id="rule">
453
+ <i style="background:var(--rose)"></i><i style="background:var(--marigold)"></i>
454
+ <i style="background:var(--sage)"></i><i style="background:var(--powder)"></i>
455
+ <i style="background:var(--lilac)"></i>
456
+ </div>
457
+ <div class="searchbar" id="term">
458
+ <span class="glyph">✦</span>
459
+ <select class="corpus-select" id="corpus" aria-label="Search corpus">
460
+ <option value="models" selected>Models</option>
461
+ <option value="kernels">Kernels</option>
462
+ <option value="docker">Docker Images</option>
463
+ </select>
464
+ <input id="q" type="text" placeholder='search by meaning… "mamba hybrid trained on code"' autocomplete="off">
465
+ <button class="btn-dig" id="go">dig ⏡</button>
466
+ </div>
467
+ <div class="status-line" id="status"></div>
468
+ <p class="promise" id="promise"><b id="hero-count">228,610</b> indie & vintage text-gen models Β· no chart-toppers, no quant reissues Β·
469
+ search by <b>meaning</b>, <b>time period</b>, <b>lineage</b>, or straight into the <b>config</b></p>
470
+ <p class="manifesto" id="manifesto">B-Sides index is more interesting than &ldquo;models nobody uses.&rdquo;
471
+ It maps the shadow infrastructure of open ML: abandoned experiments, regional
472
+ language work, architecture probes, creative utilities, and research releases that
473
+ conventional popularity rankings erase.</p>
474
+ </section>
475
+
476
+ <div id="spread" style="display:none">
477
+ <div class="crates-head">
478
+ <h3>The Spread.</h3>
479
+ <span class="sub">today's draw Β· rotates at midnight UTC</span>
480
+ </div>
481
+ <div class="spread-row" id="spread-row"></div>
482
+ </div>
483
+
484
+ <div class="crates-head" id="crates-head" style="display:none">
485
+ <h3 id="crate-title">The Reading.</h3>
486
+ <div class="sort-row">
487
+ sort
488
+ <select id="sort">
489
+ <option value="relevance">relevance</option>
490
+ <option value="newest" selected>newest</option>
491
+ <option value="oldest">oldest</option>
492
+ <option value="downloads">downloads</option>
493
+ <option value="likes">likes</option>
494
+ </select>
495
+ </div>
496
+ </div>
497
+ <div id="results-wrap">
498
+ <div class="grid" id="grid"></div>
499
+ <div class="loader"></div>
500
+ <div class="empty" id="empty">
501
+ <div class="kao">ᕦ(Λ‡Γ²_Γ³)ᕀ︻╦╀─</div>
502
+ <h4 id="empty-title">The cards are silent.</h4>
503
+ <p id="empty-copy">no record answers to that. loosen a filter or flip the query.</p>
504
+ </div>
505
+ <button class="btn-more" id="more" style="display:none">⏬ dig deeper</button>
506
+ </div>
507
+ </section>
508
+ </div>
509
+
510
+ <footer>
511
+ <span id="footer-main">pressed by <a href="https://huggingface.co/juiceb0xc0de" target="_blank">juiceb0xc0de</a> Β· slop-free: no quant reissues, no chart-toppers</span>
512
+ <span class="next" id="footer-next">next pressings: 2024 catalog Β· config-deep filters</span>
513
+ </footer>
514
+ </div>
515
+
516
+ <script>
517
+ 'use strict';
518
+ const $ = id => document.getElementById(id);
519
+ /* product defaults: the crates hide the majors and the quant reissues unless
520
+ the digger says otherwise. Checked in the UI means hidden from results. */
521
+ const DEFAULT_EXCLUDED_FAMILIES = ['llama','qwen','gemma','mistral','deepseek','phi','granite'];
522
+ const FAMILY_LABELS = {
523
+ llama: 'Llama', qwen: 'Qwen', gemma: 'Gemma', mistral: 'Mistral/Mixtral',
524
+ deepseek: 'DeepSeek', phi: 'Phi', granite: 'Granite',
525
+ };
526
+ /* mirrors CONTAINER_ARRAY_FILTERS server-side; each is its own AND group */
527
+ const DOCKER_ARRAY_FILTERS = ['base_images','accelerators','cuda_versions','rocm_versions',
528
+ 'python_versions','node_versions','operating_systems','ports','services',
529
+ 'package_managers','frameworks'];
530
+ const DOCKER_BOOL_FILTERS = ['has_compose','has_devcontainer','has_multistage'];
531
+
532
+ const state = {
533
+ corpus: 'models', page: 0, total: 0, busy: false, epoch: 0,
534
+ facets: { models: null, kernels: null, docker: null },
535
+ ready: { models: false, kernels: false, docker: false },
536
+ loading: { models: null, kernels: null, docker: null },
537
+ sel: {
538
+ models: {
539
+ archs: new Set(), methods: new Set(), imports: new Set(), moe: false, custom: false,
540
+ excludedFamilies: new Set(DEFAULT_EXCLUDED_FAMILIES),
541
+ excludeQuantizations: true,
542
+ },
543
+ kernels: {
544
+ languages: new Set(), accelerators: new Set(), cuda_archs: new Set(),
545
+ dtypes: new Set(), intrinsics: new Set(), torch_ops: new Set(),
546
+ torch_versions: new Set(), has_build_dir: false,
547
+ },
548
+ docker: {
549
+ repo_types: new Set(),
550
+ ...Object.fromEntries(DOCKER_ARRAY_FILTERS.map(k => [k, new Set()])),
551
+ has_compose: false, has_devcontainer: false, has_multistage: false,
552
+ includeArchives: false,
553
+ },
554
+ },
555
+ };
556
+
557
+ const resultKey = {models:'models', kernels:'kernels', docker:'containers'};
558
+ const endpoint = (kind, corpus) => corpus === 'models'
559
+ ? `/api/${kind}` : `/api/${kind}?corpus=${encodeURIComponent(corpus)}`;
560
+ /* card renderers are hoisted function declarations, so this map is safe here */
561
+ const CARD_RENDERERS = {models: tarotCard, kernels: kernelCard, docker: dockerCard};
562
+ const WARMING = {
563
+ models: 'warming the model crates…', kernels: 'warming the kernel crates…',
564
+ docker: 'warming the container crates…',
565
+ };
566
+ const IDLE_NOUNS = {
567
+ models: 'models in the crates', kernels: 'kernel repositories indexed',
568
+ docker: 'container artifacts indexed',
569
+ };
570
+ const RESULT_NOUNS = {models: 'models', kernels: 'kernels', docker: 'container artifacts'};
571
+ const HEAD_NOUNS = {models: 'models', kernels: 'kernels', docker: 'artifacts'};
572
+ const CRATE_TITLES = {
573
+ models: 'The Reading.', kernels: 'The Kernel Cut.', docker: 'The Build Sheet.',
574
+ };
575
+
576
+ const COPY = {
577
+ models: {
578
+ title: 'B-Sides β€” deep search for indie & vintage LLMs',
579
+ description: 'Search indie and vintage text-generation models on Hugging Face by meaning, time period, architecture, training method, lineage, and config internals.',
580
+ tagline: 'deep search Β· indie & vintage LLMs',
581
+ hero: 'The models that never charted.<br><em>Dig anyway.</em>',
582
+ placeholder: 'search by meaning… "mamba hybrid trained on code"',
583
+ promise: total => `<b id="hero-count">${total}</b> indie & vintage text-gen models Β· the majors and quant reissues are <b>hidden by default</b> β€” uncheck any of them in the rail to let them back in Β· search by <b>meaning</b>, <b>time period</b>, <b>lineage</b>, or straight into the <b>config</b>`,
584
+ manifesto: 'B-Sides index is more interesting than &ldquo;models nobody uses.&rdquo; It maps the shadow infrastructure of open ML: abandoned experiments, regional language work, architecture probes, creative utilities, and research releases that conventional popularity rankings erase.',
585
+ emptyTitle: 'The cards are silent.',
586
+ emptyCopy: 'no record answers to that. loosen a filter or flip the query.',
587
+ footer: 'pressed by <a href="https://huggingface.co/juiceb0xc0de" target="_blank">juiceb0xc0de</a> Β· slop-free: no quant reissues, no chart-toppers',
588
+ next: 'next pressings: 2024 catalog Β· config-deep filters',
589
+ catalog: '2022 pressing',
590
+ },
591
+ kernels: {
592
+ title: 'B-Sides β€” deep search for open ML kernels',
593
+ description: 'Search open ML kernel repositories on Hugging Face by meaning, language, accelerator, CUDA architecture, dtype, intrinsic, Torch operation, and build facts.',
594
+ tagline: 'deep search Β· open ML kernels',
595
+ hero: 'The kernels beneath the charts.<br><em>Dig deeper.</em>',
596
+ placeholder: 'search by meaning… "fused attention for Hopper"',
597
+ promise: total => `<b id="hero-count">${total}</b> open ML kernel repositories Β· search by <b>meaning</b>, <b>hardware path</b>, <b>dtype</b>, or <b>build facts</b>`,
598
+ manifesto: 'The kernel crates map the code beneath model releases: CUDA and Triton experiments, architecture-specific fast paths, fused operations, portability work, and optimization ideas that ordinary popularity rankings bury.',
599
+ emptyTitle: 'No kernel answered.',
600
+ emptyCopy: 'loosen a hardware filter or try another operation.',
601
+ footer: 'pressed by <a href="https://huggingface.co/juiceb0xc0de" target="_blank">juiceb0xc0de</a> Β· one repository per kernel result',
602
+ next: 'next pressing: build-graph filters',
603
+ catalog: 'kernel pressing',
604
+ },
605
+ docker: {
606
+ title: 'B-Sides β€” deep search for container recipes',
607
+ description: 'Search Docker and OCI build recipes on Hugging Face by meaning, base image, accelerator, CUDA or ROCm version, Python version, framework, service, and build facts.',
608
+ tagline: 'deep search Β· container recipes',
609
+ hero: 'The images nobody pushed.<br><em>Read the build.</em>',
610
+ placeholder: 'search by meaning… "Python 3.11 CUDA 12.8 PyTorch"',
611
+ promise: total => `<b id="hero-count">${total}</b> container artifacts Β· <b>recipes only</b> by default β€” flip <b>include archives</b> for the tarballs Β· search by <b>meaning</b>, <b>base image</b>, <b>accelerator</b>, or <b>build facts</b>`,
612
+ manifesto: 'The container crates hold the build instructions behind open ML: Dockerfiles, Compose stacks, and devcontainers that pin the exact CUDA, ROCm, Python, and framework versions a project actually ran on. It is the environment layer that model cards leave out.',
613
+ emptyTitle: 'No recipe answered.',
614
+ emptyCopy: 'loosen a build filter, or let the archives in.',
615
+ footer: 'pressed by <a href="https://huggingface.co/juiceb0xc0de" target="_blank">juiceb0xc0de</a> Β· one artifact per container result',
616
+ next: 'next pressing: layer-level build graphs',
617
+ catalog: 'container pressing',
618
+ },
619
+ };
620
+
621
+ /* pastel pairs: [top-band fill, pill fill] keyed by hash */
622
+ const PASTELS = [
623
+ ['var(--rose)','var(--rose)'], ['var(--marigold)','var(--marigold)'],
624
+ ['var(--sage)','var(--sage)'], ['var(--powder)','var(--powder)'],
625
+ ['var(--lilac)','var(--lilac)'], ['var(--terra)','var(--terra)'],
626
+ ];
627
+ const colorFor = s => PASTELS[[...s].reduce((a,c)=>a+c.charCodeAt(0),0) % PASTELS.length];
628
+ const fmtN = n => n >= 1e9 ? (n/1e9).toFixed(1).replace(/\.0$/,'')+'B'
629
+ : n >= 1e6 ? (n/1e6).toFixed(1).replace(/\.0$/,'')+'M'
630
+ : n >= 1e3 ? (n/1e3).toFixed(1).replace(/\.0$/,'')+'K' : String(n);
631
+ const esc = s => String(s).replace(/[&<>"']/g, c => ({'&':'&amp;','<':'&lt;','>':'&gt;','"':'&quot;',"'":'&#39;'}[c]));
632
+ const ROMAN = n => { // small roman numerals for card numbers
633
+ const map = [[1000,'M'],[900,'CM'],[500,'D'],[400,'CD'],[100,'C'],[90,'XC'],[50,'L'],[40,'XL'],[10,'X'],[9,'IX'],[5,'V'],[4,'IV'],[1,'I']];
634
+ let out = ''; n = (n % 3999) + 1;
635
+ for (const [v, r] of map) while (n >= v) { out += r; n -= v; }
636
+ return out;
637
+ };
638
+
639
+ function buildModelReq() {
640
+ const s = state.sel.models;
641
+ const pB = v => v ? Math.round(parseFloat(v) * 1e9) : null;
642
+ return {
643
+ query: $('q').value.trim(),
644
+ month_from: $('f-month-from').value || null,
645
+ month_to: $('f-month-to').value || null,
646
+ archs: [...s.archs], methods: [...s.methods], imports: [...s.imports],
647
+ relation: $('f-relation').value || null,
648
+ base_model: $('f-base').value.trim() || null,
649
+ license: $('f-license').value || null,
650
+ params_min: pB($('f-pmin').value), params_max: pB($('f-pmax').value),
651
+ context_min: $('f-ctx').value ? parseInt($('f-ctx').value) : null,
652
+ config_contains: $('f-config').value.trim() || null,
653
+ moe_only: s.moe, custom_code_only: s.custom,
654
+ exclude_families: [...state.sel.models.excludedFamilies],
655
+ exclude_quantizations: state.sel.models.excludeQuantizations,
656
+ sort: $('sort').value, page: state.page,
657
+ };
658
+ }
659
+
660
+ /* an inverted size range can never match anything β€” say so instead of asking
661
+ the backend for the empty set */
662
+ function sizeRangeInvalid() {
663
+ const min = parseFloat($('f-pmin').value);
664
+ const max = parseFloat($('f-pmax').value);
665
+ const bad = Number.isFinite(min) && Number.isFinite(max) && min > max;
666
+ $('size-warn').style.display = bad ? '' : 'none';
667
+ return bad;
668
+ }
669
+
670
+ function buildDockerReq() {
671
+ const s = state.sel.docker;
672
+ const filters = {};
673
+ const put = (key, values) => { if (values.length) filters[key] = values; };
674
+ // Recipes are the product default; the archive toggle drops the constraint
675
+ // rather than widening it to a second kind.
676
+ if (!s.includeArchives) filters.artifact_kinds = ['recipe'];
677
+ put('repo_types', [...s.repo_types]);
678
+ for (const key of DOCKER_ARRAY_FILTERS) put(key, [...s[key]]);
679
+ for (const key of DOCKER_BOOL_FILTERS) if (s[key]) filters[key] = true;
680
+ return {
681
+ query: $('q').value.trim(), sort: $('sort').value,
682
+ page: state.page, filters,
683
+ };
684
+ }
685
+
686
+ function buildKernelReq() {
687
+ const s = state.sel.kernels;
688
+ const filters = {};
689
+ const put = (key, value) => {
690
+ if (value !== '' && value != null && (!Array.isArray(value) || value.length))
691
+ filters[key] = value;
692
+ };
693
+ put('month_from', $('k-month-from').value);
694
+ put('month_to', $('k-month-to').value);
695
+ put('author', $('k-author').value);
696
+ put('downloads_min', $('k-downloads-min').value ? parseInt($('k-downloads-min').value) : null);
697
+ put('downloads_max', $('k-downloads-max').value ? parseInt($('k-downloads-max').value) : null);
698
+ put('variant_count_min', $('k-variant-min').value ? parseInt($('k-variant-min').value) : null);
699
+ for (const key of ['languages','accelerators','cuda_archs','dtypes','intrinsics','torch_ops','torch_versions'])
700
+ put(key, [...s[key]]);
701
+ if (s.has_build_dir) filters.has_build_dir = true;
702
+ return {
703
+ query: $('q').value.trim(), sort: $('sort').value,
704
+ page: state.page, filters,
705
+ };
706
+ }
707
+
708
+ const REQ_BUILDERS = {models: buildModelReq, kernels: buildKernelReq, docker: buildDockerReq};
709
+ const buildReq = () => REQ_BUILDERS[state.corpus]();
710
+
711
+ // default exclusions are product defaults, not user intent β€” they leave the
712
+ // landing state pristine, same as the default Docker recipe constraint.
713
+ const modelIsPristine = r => !r.query && !r.month_from && !r.month_to && !r.archs.length
714
+ && !r.methods.length && !r.imports.length && !r.relation && !r.base_model
715
+ && !r.license && r.params_min == null && r.params_max == null
716
+ && r.context_min == null && !r.config_contains && !r.moe_only && !r.custom_code_only;
717
+ const kernelIsPristine = r => !r.query && Object.keys(r.filters).length === 0;
718
+ const dockerIsPristine = r => {
719
+ if (r.query) return false;
720
+ const keys = Object.keys(r.filters);
721
+ return keys.length === 0 || (keys.length === 1 && keys[0] === 'artifact_kinds');
722
+ };
723
+ const PRISTINE = {models: modelIsPristine, kernels: kernelIsPristine, docker: dockerIsPristine};
724
+ const isPristine = r => PRISTINE[state.corpus](r);
725
+
726
+ function tarotCard(m) {
727
+ const archTag = m.model_type || m.tags.find(t => state.facets.models._archset.has(t)) || '';
728
+ const [band] = colorFor(archTag || m.model_id);
729
+ const method = ['grpo','dpo','orpo','kto','ppo','sft','lora','merge'].find(t => m.tags.includes(t));
730
+
731
+ const pills = [];
732
+ for (const imp of (m.code_imports || []).filter(i => ['flash_attn','causal_conv1d','mamba_ssm','triton','fla'].includes(i)))
733
+ pills.push(`<span class="pill hot">${esc(imp)}</span>`);
734
+ if (m.gated) pills.push(`<span class="pill" style="--pc:var(--lilac)">gated</span>`);
735
+ if (m.tags.includes('unsloth')) pills.push(`<span class="pill" style="--pc:var(--sage)">unsloth</span>`);
736
+
737
+ const meta = [`<span>${esc(m.month || '')}</span>`];
738
+ if (m.params) meta.push(`<span>${fmtN(m.params)} params</span>`);
739
+ if (m.context_len) meta.push(`<span>${fmtN(m.context_len)} ctx</span>`);
740
+ meta.push(`<span>↓${fmtN(m.downloads||0)} β™₯${fmtN(m.likes||0)}</span>`);
741
+ if (m.score != null) meta.push(`<span class="sc-match">${(m.score*100).toFixed(0)}%</span>`);
742
+
743
+ const lineage = m.base_model
744
+ ? `<div class="lineage">${esc(m.relation || 'b-side')} of ${esc(m.base_model)}</div>` : '';
745
+ const caption = [archTag, method].filter(Boolean).join(' Β· ') || 'text generation';
746
+
747
+ const [author, ...rest] = m.model_id.split('/');
748
+ return `<div class="tcard" style="--tc:${band}" data-href="https://huggingface.co/${encodeURIComponent(author)}/${rest.map(encodeURIComponent).join('/')}">
749
+ <div class="sky"><span class="num">β„– ${ROMAN(m.row)}</span><span>${esc(m.month || '')}</span></div>
750
+ <div class="body">
751
+ <div class="mid">${esc(rest.join('/') || m.model_id)}</div>
752
+ <div class="author">${esc(author)}</div>
753
+ <div class="meta">${meta.join('')}</div>
754
+ ${pills.length ? `<div class="pills">${pills.join('')}</div>` : ''}
755
+ ${lineage}
756
+ </div>
757
+ <div class="caption">${esc(caption)}&nbsp;.</div>
758
+ </div>`;
759
+ }
760
+
761
+ function kernelCard(k) {
762
+ const selected = state.sel.kernels;
763
+ const [authorFromId, ...rest] = k.repo_id.split('/');
764
+ const author = k.author || authorFromId;
765
+ const name = rest.join('/') || k.repo_id;
766
+ const [band] = colorFor((k.accelerators || [])[0] || k.repo_id);
767
+ const preferSelected = (values, set, limit) => [...(values || [])]
768
+ .sort((a, b) => Number(set.has(b)) - Number(set.has(a))).slice(0, limit);
769
+
770
+ const pills = [];
771
+ for (const value of [...(k.languages || []), ...(k.accelerators || [])].slice(0, 4))
772
+ pills.push(`<span class="pill hot">${esc(value)}</span>`);
773
+ for (const value of (k.dtypes || []).slice(0, 2))
774
+ pills.push(`<span class="pill" style="--pc:var(--lilac)">${esc(value)}</span>`);
775
+ for (const value of preferSelected(k.intrinsics, selected.intrinsics, 2))
776
+ pills.push(`<span class="pill" style="--pc:var(--terra)">${esc(value)}</span>`);
777
+ for (const value of preferSelected(k.torch_ops, selected.torch_ops, 2))
778
+ pills.push(`<span class="pill" style="--pc:var(--sage)">${esc(value)}</span>`);
779
+
780
+ const meta = [`<span>${esc(k.month || '')}</span>`,
781
+ `<span>↓${fmtN(k.downloads||0)} β™₯${fmtN(k.likes||0)}</span>`];
782
+ if (k.score != null) meta.push(`<span class="sc-match">${(k.score*100).toFixed(0)}%</span>`);
783
+ const variants = `${fmtN(k.variant_count || 0)} variant${k.variant_count === 1 ? '' : 's'}`;
784
+ const sources = `${fmtN(k.source_files || 0)} source file${k.source_files === 1 ? '' : 's'}`;
785
+ const href = `https://huggingface.co/${k.repo_id.split('/').map(encodeURIComponent).join('/')}`;
786
+
787
+ return `<div class="tcard kernel-card" style="--tc:${band}" data-href="${href}">
788
+ <div class="sky"><span class="num">β„– ${ROMAN(k.row)}</span><span>${esc(k.month || '')}</span></div>
789
+ <div class="body">
790
+ <div class="mid">${esc(name)}</div>
791
+ <div class="author">${esc(author)}</div>
792
+ <div class="meta">${meta.join('')}</div>
793
+ ${pills.length ? `<div class="pills">${pills.slice(0, 10).join('')}</div>` : ''}
794
+ <div class="facts">${variants} Β· ${sources}</div>
795
+ </div>
796
+ <div class="caption">kernel repository&nbsp;.</div>
797
+ </div>`;
798
+ }
799
+
800
+ /* Buckets have no git revisions, and recipe rows carry the synthetic
801
+ `<recipe>` path rather than a real file β€” neither has a valid blob URL, so
802
+ those land on the repository page instead of a guaranteed 404. */
803
+ function containerHref(c) {
804
+ const repoType = c.repo_type || 'model';
805
+ const path = c.artifact_path || '';
806
+ const repo = String(c.repo_id || '').split('/').map(encodeURIComponent).join('/');
807
+ if (repoType === 'bucket') return `https://huggingface.co/buckets/${repo}`;
808
+ const prefix = repoType === 'space' ? 'spaces/' : repoType === 'dataset' ? 'datasets/' : '';
809
+ const landing = `https://huggingface.co/${prefix}${repo}`;
810
+ if (!c.sha || !path || path === '<recipe>') return landing;
811
+ const file = path.split('/').map(encodeURIComponent).join('/');
812
+ return `${landing}/blob/${encodeURIComponent(c.sha)}/${file}`;
813
+ }
814
+
815
+ function dockerCard(c) {
816
+ const selected = state.sel.docker;
817
+ const [authorFromId, ...rest] = String(c.repo_id || '').split('/');
818
+ const author = c.author || authorFromId;
819
+ const name = rest.join('/') || c.repo_id || '';
820
+ const path = c.artifact_path || '';
821
+ const kind = c.artifact_kind || 'recipe';
822
+ const month = (c.created_at || '').slice(0, 7);
823
+ const [band] = colorFor((c.accelerators || [])[0] || (c.base_images || [])[0] || name);
824
+ const preferSelected = (values, set, limit) => [...(values || [])]
825
+ .sort((a, b) => Number(set.has(b)) - Number(set.has(a))).slice(0, limit);
826
+
827
+ const pills = [];
828
+ for (const value of (c.accelerators || []).slice(0, 2))
829
+ pills.push(`<span class="pill hot">${esc(value)}</span>`);
830
+ for (const value of preferSelected(c.cuda_versions, selected.cuda_versions, 2))
831
+ pills.push(`<span class="pill" style="--pc:var(--powder)">cuda ${esc(value)}</span>`);
832
+ for (const value of preferSelected(c.rocm_versions, selected.rocm_versions, 1))
833
+ pills.push(`<span class="pill" style="--pc:var(--terra)">rocm ${esc(value)}</span>`);
834
+ for (const value of preferSelected(c.python_versions, selected.python_versions, 2))
835
+ pills.push(`<span class="pill" style="--pc:var(--marigold)">py ${esc(value)}</span>`);
836
+ for (const value of preferSelected(c.frameworks, selected.frameworks, 3))
837
+ pills.push(`<span class="pill" style="--pc:var(--sage)">${esc(value)}</span>`);
838
+ if (c.has_compose) pills.push(`<span class="pill" style="--pc:var(--lilac)">compose</span>`);
839
+ if (c.has_devcontainer) pills.push(`<span class="pill" style="--pc:var(--lilac)">devcontainer</span>`);
840
+ if (c.has_multistage) pills.push(`<span class="pill" style="--pc:var(--lilac)">multi-stage</span>`);
841
+
842
+ const meta = [`<span>${esc(month)}</span>`,
843
+ `<span>↓${fmtN(c.downloads||0)} β™₯${fmtN(c.likes||0)}</span>`];
844
+ if (c.score != null) meta.push(`<span class="sc-match">${(c.score*100).toFixed(0)}%</span>`);
845
+
846
+ const base = (c.base_images || [])[0];
847
+ const facts = [base ? `from ${esc(base)}` : '',
848
+ path && path !== '<recipe>' ? esc(path) : esc(kind)].filter(Boolean).join(' Β· ');
849
+ const caption = [kind, c.repo_type].filter(Boolean).join(' Β· ') || 'container artifact';
850
+
851
+ return `<div class="tcard docker-card" style="--tc:${band}" data-href="${esc(containerHref(c))}">
852
+ <div class="sky"><span class="num">β„– ${ROMAN(c.row)}</span><span>${esc(month)}</span></div>
853
+ <div class="body">
854
+ <div class="mid">${esc(name)}</div>
855
+ <div class="author">${esc(author)}</div>
856
+ <div class="meta">${meta.join('')}</div>
857
+ ${pills.length ? `<div class="pills">${pills.slice(0, 10).join('')}</div>` : ''}
858
+ <div class="facts">${facts}</div>
859
+ </div>
860
+ <div class="caption">${esc(caption)}&nbsp;.</div>
861
+ </div>`;
862
+ }
863
+
864
+ function wireCards(scope) {
865
+ const fresh = [...scope.querySelectorAll('.tcard:not(.wired)')];
866
+ fresh.forEach(el => {
867
+ el.classList.add('wired');
868
+ el.addEventListener('click', () => window.open(el.dataset.href, '_blank'));
869
+ });
870
+ if (window.VanillaTilt) VanillaTilt.init(fresh, { max: 4, speed: 900, scale: 1.015, gyroscope: false });
871
+ if (window.gsap) gsap.from(fresh, { y: 16, opacity: 0, duration: .45, stagger: 0.04, ease: 'power3.out', clearProps: 'opacity,transform' });
872
+ }
873
+
874
+ // boot window: the backend returns 503 while the ~1.4 GB index streams in off
875
+ // disk β€” retry politely instead of showing a broken page.
876
+ async function getJSON(url, opts, tries = 60, delayMs = 1000) {
877
+ for (let i = 0; ; i++) {
878
+ const r = await fetch(url, opts);
879
+ if (r.ok) return r.json();
880
+ if (r.status !== 503 || i >= tries)
881
+ throw new Error((await r.json().catch(() => ({}))).detail
882
+ || `request failed (${r.status})`);
883
+ await new Promise(res => setTimeout(res, delayMs));
884
+ }
885
+ }
886
+
887
+ async function loadSpread() {
888
+ try {
889
+ const d = await getJSON('/api/picks');
890
+ if (!d.picks.length) return;
891
+ $('spread-row').innerHTML = d.picks.map(p => `
892
+ <div class="pick">
893
+ ${tarotCard(p)}
894
+ <div class="note">β€œ${esc(p.note)}”</div>
895
+ <div class="by">β€” ${esc(p.picked_by)}</div>
896
+ </div>`).join('');
897
+ wireCards($('spread-row'));
898
+ if (state.corpus === 'models') $('spread').style.display = '';
899
+ } catch (e) { /* picks are optional */ }
900
+ }
901
+
902
+ function idle() {
903
+ /* landing state: search engine at rest β€” spread only, no listing */
904
+ $('grid').innerHTML = '';
905
+ $('crates-head').style.display = 'none';
906
+ $('empty').style.display = 'none';
907
+ $('more').style.display = 'none';
908
+ $('spread').style.display = state.corpus === 'models' && $('spread-row').children.length ? '' : 'none';
909
+ const facets = state.facets[state.corpus];
910
+ $('status').innerHTML = facets
911
+ ? `<b>${facets.total.toLocaleString()}</b> ${IDLE_NOUNS[state.corpus]}` : '';
912
+ }
913
+
914
+ async function search(append=false) {
915
+ if (state.busy) return;
916
+ if (!state.ready[state.corpus]) {
917
+ $('status').textContent = WARMING[state.corpus];
918
+ return;
919
+ }
920
+ if (state.corpus === 'models' && sizeRangeInvalid()) return;
921
+ const reqCheck = buildReq();
922
+ if (isPristine(reqCheck)) { idle(); return; }
923
+ const corpus = state.corpus;
924
+ const epoch = state.epoch;
925
+ state.busy = true;
926
+ if (!append) { state.page = 0; $('grid').innerHTML = ''; }
927
+ $('results-wrap').parentElement.classList.add('loading');
928
+ $('empty').style.display = 'none';
929
+ try {
930
+ const req = buildReq();
931
+ const data = await getJSON(endpoint('search', corpus), { method: 'POST',
932
+ headers: {'Content-Type':'application/json'}, body: JSON.stringify(req) }, 60, 1000);
933
+ if (corpus !== state.corpus || epoch !== state.epoch) return;
934
+ state.total = data.total;
935
+ const rows = data[resultKey[corpus]] || [];
936
+ const html = rows.map(CARD_RENDERERS[corpus]).join('');
937
+ if (append) $('grid').insertAdjacentHTML('beforeend', html);
938
+ else $('grid').innerHTML = html;
939
+ wireCards($('grid'));
940
+
941
+ $('spread').style.display = 'none';
942
+ $('crates-head').style.display = '';
943
+ const noun = RESULT_NOUNS[corpus];
944
+ $('status').innerHTML = `<b>${data.total.toLocaleString()}</b> ${noun}` +
945
+ (req.query && req.sort === 'relevance' ? ` Β· matched by meaning to β€œ${esc(req.query)}”` : ' match your filters');
946
+ $('empty').style.display = data.total === 0 ? 'block' : 'none';
947
+ $('more').style.display = $('grid').children.length < data.total ? 'block' : 'none';
948
+ } catch (e) {
949
+ if (corpus === state.corpus && epoch === state.epoch)
950
+ $('status').innerHTML = `⚠ ${esc(e.message)}`;
951
+ } finally {
952
+ if (corpus === state.corpus && epoch === state.epoch) {
953
+ $('results-wrap').parentElement.classList.remove('loading');
954
+ state.busy = false;
955
+ }
956
+ }
957
+ }
958
+
959
+ function chipbar(el, items, set, max=14) {
960
+ const render = (expanded) => {
961
+ const list = expanded ? items : items.slice(0, max);
962
+ el.innerHTML = list.map(i =>
963
+ `<span class="chip${set.has(i.v)?' on':''}" data-v="${esc(i.v)}">${esc(i.v)}<span class="n">${fmtN(i.n)}</span></span>`
964
+ ).join('') + (items.length > max
965
+ ? `<span class="chip more-toggle">${expanded ? 'βˆ’ less' : `+ ${items.length-max} more`}</span>` : '');
966
+ el.querySelectorAll('.chip').forEach(c => c.onclick = () => {
967
+ if (c.classList.contains('more-toggle')) return render(!expanded);
968
+ const v = c.dataset.v;
969
+ set.has(v) ? set.delete(v) : set.add(v);
970
+ c.classList.toggle('on');
971
+ search();
972
+ });
973
+ };
974
+ render(false);
975
+ }
976
+
977
+ function monthOptions(from, to, months) {
978
+ $(from).innerHTML = '<option value="">from the start</option>' +
979
+ months.map(m=>`<option>${m}</option>`).join('');
980
+ $(to).innerHTML = '<option value="">to the latest</option>' +
981
+ months.map(m=>`<option>${m}</option>`).join('');
982
+ }
983
+
984
+ function populateModelFacets(f) {
985
+ f._archset = new Set(f.archs.map(a => a.v));
986
+ monthOptions('f-month-from', 'f-month-to', f.months);
987
+ chipbar($('f-archs'), f.archs, state.sel.models.archs, 6);
988
+ chipbar($('f-methods'), f.methods, state.sel.models.methods, 6);
989
+ $('f-relation').innerHTML = '<option value="">any relation</option>' +
990
+ f.relations.map(r=>`<option value="${esc(r.v)}">${esc(r.v)} (${fmtN(r.n)})</option>`).join('');
991
+ $('f-license').innerHTML = '<option value="">any license</option>' +
992
+ f.licenses.map(l=>`<option value="${esc(l.v)}">${esc(l.v)} (${fmtN(l.n)})</option>`).join('');
993
+ if (f.enriched) {
994
+ if (f.imports.length) chipbar($('f-imports'), f.imports, state.sel.models.imports, 10);
995
+ } else {
996
+ $('deep-soon').style.display = '';
997
+ $('deep-filters').style.display = 'none';
998
+ }
999
+ }
1000
+
1001
+ function populateKernelFacets(f) {
1002
+ monthOptions('k-month-from', 'k-month-to', f.months);
1003
+ for (const [key, id, max] of [
1004
+ ['languages','k-languages',10], ['accelerators','k-accelerators',8],
1005
+ ['cuda_archs','k-cuda-archs',10], ['dtypes','k-dtypes',10],
1006
+ ['intrinsics','k-intrinsics',10], ['torch_ops','k-torch-ops',10],
1007
+ ['torch_versions','k-torch-versions',10],
1008
+ ]) chipbar($(id), f[key] || [], state.sel.kernels[key], max);
1009
+ $('k-author').innerHTML = '<option value="">any author</option>' +
1010
+ (f.authors || []).map(a=>`<option value="${esc(a.v)}">${esc(a.v)} (${fmtN(a.n)})</option>`).join('');
1011
+ }
1012
+
1013
+ /* checked means hidden. These are product defaults, so they render on; the
1014
+ list is fixed by DEFAULT_EXCLUDED_FAMILIES and stays visible once unchecked. */
1015
+ function renderExclusions() {
1016
+ const s = state.sel.models;
1017
+ const rows = [['__quant', 'Quantizations', s.excludeQuantizations]].concat(
1018
+ DEFAULT_EXCLUDED_FAMILIES.map(f => [f, FAMILY_LABELS[f] || f, s.excludedFamilies.has(f)]));
1019
+ $('f-excludes').innerHTML = rows.map(([key, label, on]) =>
1020
+ `<span class="ex-row${on ? ' on' : ''}" data-k="${esc(key)}"><span class="box"></span>${esc(label)}</span>`
1021
+ ).join('');
1022
+ $('f-excludes').querySelectorAll('.ex-row').forEach(row => row.onclick = () => {
1023
+ const key = row.dataset.k;
1024
+ if (key === '__quant') s.excludeQuantizations = !s.excludeQuantizations;
1025
+ else s.excludedFamilies.has(key)
1026
+ ? s.excludedFamilies.delete(key) : s.excludedFamilies.add(key);
1027
+ row.classList.toggle('on');
1028
+ search();
1029
+ });
1030
+ }
1031
+
1032
+ function populateDockerFacets(f) {
1033
+ const s = state.sel.docker;
1034
+ chipbar($('d-repo-types'), f.repo_types || [], s.repo_types, 6);
1035
+ for (const [key, id, max] of [
1036
+ ['accelerators','d-accelerators',8], ['cuda_versions','d-cuda-versions',10],
1037
+ ['rocm_versions','d-rocm-versions',8], ['python_versions','d-python-versions',10],
1038
+ ['frameworks','d-frameworks',10], ['base_images','d-base-images',10],
1039
+ ['operating_systems','d-operating-systems',8],
1040
+ ['package_managers','d-package-managers',8], ['services','d-services',10],
1041
+ ['node_versions','d-node-versions',8],
1042
+ ]) chipbar($(id), f[key] || [], s[key], max);
1043
+ }
1044
+
1045
+ const FACET_POPULATORS = {
1046
+ models: populateModelFacets, kernels: populateKernelFacets,
1047
+ docker: populateDockerFacets,
1048
+ };
1049
+
1050
+ function applyCopy() {
1051
+ const copy = COPY[state.corpus];
1052
+ const facets = state.facets[state.corpus];
1053
+ const total = facets ? facets.total.toLocaleString() : '…';
1054
+ document.title = copy.title;
1055
+ document.querySelector('meta[name="description"]').content = copy.description;
1056
+ $('tagline').textContent = copy.tagline;
1057
+ $('hero-h').innerHTML = copy.hero;
1058
+ $('q').placeholder = copy.placeholder;
1059
+ $('promise').innerHTML = copy.promise(total);
1060
+ $('manifesto').innerHTML = copy.manifesto;
1061
+ $('empty-title').textContent = copy.emptyTitle;
1062
+ $('empty-copy').textContent = copy.emptyCopy;
1063
+ $('footer-main').innerHTML = copy.footer;
1064
+ $('footer-next').textContent = copy.next;
1065
+ $('catalog-copy').textContent = copy.catalog;
1066
+ $('head-count').textContent = facets
1067
+ ? `${total} ${HEAD_NOUNS[state.corpus]}` : '…';
1068
+ $('crate-title').textContent = CRATE_TITLES[state.corpus];
1069
+ }
1070
+
1071
+ async function loadFacets(corpus) {
1072
+ if (state.ready[corpus]) return state.facets[corpus];
1073
+ if (state.loading[corpus]) return state.loading[corpus];
1074
+ state.loading[corpus] = getJSON(endpoint('facets', corpus)).then(f => {
1075
+ state.facets[corpus] = f;
1076
+ state.ready[corpus] = true;
1077
+ FACET_POPULATORS[corpus](f);
1078
+ return f;
1079
+ }).finally(() => { state.loading[corpus] = null; });
1080
+ return state.loading[corpus];
1081
+ }
1082
+
1083
+ function resetResults() {
1084
+ state.page = 0;
1085
+ state.total = 0;
1086
+ state.busy = false;
1087
+ $('grid').innerHTML = '';
1088
+ $('crates-head').style.display = 'none';
1089
+ $('empty').style.display = 'none';
1090
+ $('more').style.display = 'none';
1091
+ $('results-wrap').parentElement.classList.remove('loading');
1092
+ }
1093
+
1094
+ async function switchCorpus(corpus) {
1095
+ state.corpus = corpus;
1096
+ state.epoch++;
1097
+ resetResults();
1098
+ $('model-filters').style.display = corpus === 'models' ? '' : 'none';
1099
+ $('kernel-filters').style.display = corpus === 'kernels' ? '' : 'none';
1100
+ $('docker-filters').style.display = corpus === 'docker' ? '' : 'none';
1101
+ $('spread').style.display = 'none';
1102
+ applyCopy();
1103
+ $('status').textContent = WARMING[corpus];
1104
+ try {
1105
+ await loadFacets(corpus);
1106
+ if (state.corpus !== corpus) return;
1107
+ applyCopy();
1108
+ idle();
1109
+ } catch (e) {
1110
+ if (state.corpus === corpus)
1111
+ $('status').innerHTML = `⚠ ${esc(e.message)}`;
1112
+ }
1113
+ }
1114
+
1115
+ function clearModels() {
1116
+ for (const id of ['f-month-from','f-month-to','f-relation','f-license']) $(id).value = '';
1117
+ for (const id of ['f-base','f-pmin','f-pmax','f-ctx','f-config']) $(id).value = '';
1118
+ const s = state.sel.models;
1119
+ s.archs.clear(); s.methods.clear(); s.imports.clear();
1120
+ s.moe = s.custom = false;
1121
+ $('f-moe').classList.remove('on'); $('f-custom').classList.remove('on');
1122
+ $('model-filters').querySelectorAll('.chip.on').forEach(c => c.classList.remove('on'));
1123
+ // clearing restores the product defaults; it does not switch them off, which
1124
+ // would silently widen every search that follows.
1125
+ s.excludedFamilies = new Set(DEFAULT_EXCLUDED_FAMILIES);
1126
+ s.excludeQuantizations = true;
1127
+ $('size-warn').style.display = 'none';
1128
+ renderExclusions();
1129
+ }
1130
+
1131
+ function clearKernels() {
1132
+ for (const id of ['k-month-from','k-month-to','k-author']) $(id).value = '';
1133
+ for (const id of ['k-downloads-min','k-downloads-max','k-variant-min']) $(id).value = '';
1134
+ const s = state.sel.kernels;
1135
+ for (const key of ['languages','accelerators','cuda_archs','dtypes','intrinsics','torch_ops','torch_versions'])
1136
+ s[key].clear();
1137
+ s.has_build_dir = false;
1138
+ $('k-build').classList.remove('on');
1139
+ $('kernel-filters').querySelectorAll('.chip.on').forEach(c => c.classList.remove('on'));
1140
+ }
1141
+
1142
+ function clearDocker() {
1143
+ const s = state.sel.docker;
1144
+ s.repo_types.clear();
1145
+ for (const key of DOCKER_ARRAY_FILTERS) s[key].clear();
1146
+ for (const key of DOCKER_BOOL_FILTERS) s[key] = false;
1147
+ s.includeArchives = false; // back to recipes only
1148
+ for (const id of ['d-compose','d-devcontainer','d-multistage','d-archives'])
1149
+ $(id).classList.remove('on');
1150
+ $('docker-filters').querySelectorAll('.chip.on').forEach(c => c.classList.remove('on'));
1151
+ }
1152
+
1153
+ const CLEARERS = {models: clearModels, kernels: clearKernels, docker: clearDocker};
1154
+
1155
+ async function init() {
1156
+ applyCopy();
1157
+ renderExclusions();
1158
+ $('status').textContent = WARMING.models;
1159
+ try {
1160
+ await loadFacets('models');
1161
+ applyCopy();
1162
+ await loadSpread();
1163
+ idle();
1164
+ } catch (e) {
1165
+ $('status').innerHTML = `⚠ ${esc(e.message)}`;
1166
+ }
1167
+ }
1168
+
1169
+ /* entrance: quiet, three beats */
1170
+ if (window.gsap) {
1171
+ gsap.from('#hdr', { y: -14, opacity: 0, duration: .5, ease: 'power2.out' });
1172
+ gsap.from('#hero-h', { y: 20, opacity: 0, duration: .6, delay: .1, ease: 'power3.out' });
1173
+ gsap.from('#rule i', { scaleX: 0, duration: .4, stagger: .06, delay: .35, ease: 'power2.out' });
1174
+ gsap.from('#term', { y: 14, opacity: 0, duration: .5, delay: .45, ease: 'power2.out' });
1175
+ }
1176
+
1177
+ /* events */
1178
+ $('go').onclick = () => { if ($('q').value.trim()) $('sort').value = 'relevance'; search(); };
1179
+ $('q').addEventListener('keydown', e => { if (e.key === 'Enter') $('go').click(); });
1180
+ $('corpus').onchange = () => switchCorpus($('corpus').value);
1181
+ $('more').onclick = () => { state.page++; search(true); };
1182
+ $('sort').onchange = () => search();
1183
+ for (const id of ['f-month-from','f-month-to','f-relation','f-license'])
1184
+ $(id).onchange = () => search();
1185
+ let deb;
1186
+ for (const id of ['f-base','f-pmin','f-pmax','f-ctx','f-config'])
1187
+ $(id).addEventListener('input', () => { clearTimeout(deb); deb = setTimeout(() => search(), 450); });
1188
+ for (const [id, key] of [['f-moe','moe'],['f-custom','custom']])
1189
+ $(id).onclick = () => { const s = state.sel.models; s[key] = !s[key]; $(id).classList.toggle('on'); search(); };
1190
+ for (const id of ['k-month-from','k-month-to','k-author'])
1191
+ $(id).onchange = () => search();
1192
+ for (const id of ['k-downloads-min','k-downloads-max','k-variant-min'])
1193
+ $(id).addEventListener('input', () => { clearTimeout(deb); deb = setTimeout(() => search(), 450); });
1194
+ $('k-build').onclick = () => {
1195
+ const s = state.sel.kernels;
1196
+ s.has_build_dir = !s.has_build_dir;
1197
+ $('k-build').classList.toggle('on');
1198
+ search();
1199
+ };
1200
+ for (const [id, key] of [['d-compose','has_compose'],['d-devcontainer','has_devcontainer'],
1201
+ ['d-multistage','has_multistage'],['d-archives','includeArchives']])
1202
+ $(id).onclick = () => {
1203
+ const s = state.sel.docker;
1204
+ s[key] = !s[key];
1205
+ $(id).classList.toggle('on');
1206
+ search();
1207
+ };
1208
+ $('clear').onclick = () => {
1209
+ $('q').value = ''; $('sort').value = 'newest';
1210
+ CLEARERS[state.corpus]();
1211
+ search();
1212
+ };
1213
+
1214
+ init();
1215
+ </script>
1216
+ </body>
1217
+ </html>
tests/.DS_Store ADDED
Binary file (6.15 kB). View file
 
tests/test_config.py ADDED
@@ -0,0 +1,127 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ from pathlib import Path
2
+
3
+ import pytest
4
+
5
+ from bsides.config import Settings
6
+
7
+
8
+ def test_defaults_match_the_production_crawl_contract() -> None:
9
+ settings = Settings.from_env(
10
+ {"HF_TOKEN": "present", "OPENAI_API_KEY": "present"}
11
+ )
12
+
13
+ assert settings.pipelines == (
14
+ "text-generation",
15
+ "text2text-generation",
16
+ )
17
+ assert settings.hub_requests_per_window == 2_400
18
+ assert settings.hub_window_seconds == 300.0
19
+ assert settings.embedding_model == "text-embedding-3-small"
20
+ assert settings.embedding_dimensions == 256
21
+ assert settings.embedding_tokens_per_minute == 4_500_000
22
+ assert settings.chunk_target_tokens == 1_200
23
+ assert settings.chunk_overlap_tokens == 150
24
+ assert settings.max_file_bytes == 2 * 1024 * 1024
25
+ assert settings.max_repo_bytes == 16 * 1024 * 1024
26
+ assert settings.checkpoint_every_repos == 2_500
27
+ assert settings.fetch_queue_multiplier == 4
28
+ assert settings.embed_workers == 8
29
+ assert settings.checkpoint_interval_seconds == 1_800.0
30
+ assert settings.progress_interval_seconds == 10.0
31
+ assert settings.tail_interval_seconds == 900.0
32
+ assert settings.tail_overlap_ids == 100
33
+ assert settings.dataset_repo_id == "juiceb0xc0de/b-sides-v2-index"
34
+ assert settings.run_id == "v2"
35
+ assert settings.lease_ttl_seconds == 1_800.0
36
+ assert settings.state_dir == Path("/tmp/bsides")
37
+ assert settings.mode == "both"
38
+
39
+
40
+ def test_crawl_end_is_an_explicit_inclusive_upper_boundary() -> None:
41
+ settings = Settings.from_env(
42
+ {"BSIDES_CRAWL_END": "2026-08-30T06:00:00Z"}
43
+ )
44
+
45
+ assert settings.crawl_end == "2026-08-30T06:00:00Z"
46
+
47
+
48
+ def test_environment_overrides_are_parsed_without_storing_secret_values() -> None:
49
+ settings = Settings.from_env(
50
+ {
51
+ "HF_TOKEN": "hf-secret",
52
+ "OPENAI_API_KEY": "openai-secret",
53
+ "BSIDES_MODE": "collect",
54
+ "BSIDES_HUB_REQUESTS_PER_5M": "1234",
55
+ "BSIDES_CHECKPOINT_EVERY_REPOS": "17",
56
+ "BSIDES_FETCH_QUEUE_MULTIPLIER": "3",
57
+ "BSIDES_EMBED_WORKERS": "5",
58
+ "BSIDES_EMBEDDING_TPM": "4000000",
59
+ "BSIDES_CHECKPOINT_INTERVAL_SECONDS": "45",
60
+ "BSIDES_PROGRESS_INTERVAL_SECONDS": "7",
61
+ "BSIDES_TAIL_INTERVAL_SECONDS": "60",
62
+ "BSIDES_STATE_DIR": "/tmp/custom-b-sides",
63
+ "BSIDES_DATASET": "owner/index",
64
+ "BSIDES_YEAR": "2025",
65
+ "BSIDES_CRAWL_END": "2026-08-30T06:00:00Z",
66
+ }
67
+ )
68
+
69
+ assert settings.mode == "collect"
70
+ assert settings.hub_requests_per_window == 1_234
71
+ assert settings.checkpoint_every_repos == 17
72
+ assert settings.fetch_queue_multiplier == 3
73
+ assert settings.embed_workers == 5
74
+ assert settings.embedding_tokens_per_minute == 4_000_000
75
+ assert settings.checkpoint_interval_seconds == 45.0
76
+ assert settings.progress_interval_seconds == 7.0
77
+ assert settings.tail_interval_seconds == 60.0
78
+ assert settings.state_dir == Path("/tmp/custom-b-sides")
79
+ assert settings.dataset_repo_id == "owner/index"
80
+ assert settings.crawl_year == 2025
81
+ assert settings.crawl_end == "2026-08-30T06:00:00Z"
82
+ assert settings.hf_token_present is True
83
+ assert settings.openai_api_key_present is True
84
+ assert "hf-secret" not in repr(settings)
85
+ assert "openai-secret" not in repr(settings)
86
+
87
+
88
+ @pytest.mark.parametrize(
89
+ ("env", "message"),
90
+ [
91
+ ({"BSIDES_MODE": "wrong"}, "BSIDES_MODE"),
92
+ ({"BSIDES_HUB_REQUESTS_PER_5M": "0"}, "Hub request budget"),
93
+ ({"BSIDES_HUB_REQUESTS_PER_5M": "3000"}, "Hub request budget"),
94
+ ({"BSIDES_EMBEDDING_DIMENSIONS": "0"}, "embedding dimensions"),
95
+ ({"BSIDES_CHUNK_OVERLAP_TOKENS": "1200"}, "overlap"),
96
+ ({"BSIDES_FETCH_QUEUE_MULTIPLIER": "0"}, "queue multiplier"),
97
+ ({"BSIDES_EMBED_WORKERS": "0"}, "embedding workers"),
98
+ ({"BSIDES_EMBEDDING_TPM": "0"}, "embedding token rate"),
99
+ ({"BSIDES_CHECKPOINT_INTERVAL_SECONDS": "0"}, "checkpoint interval"),
100
+ ({"BSIDES_PROGRESS_INTERVAL_SECONDS": "0"}, "progress interval"),
101
+ ],
102
+ )
103
+ def test_invalid_settings_are_rejected(env: dict[str, str], message: str) -> None:
104
+ with pytest.raises(ValueError, match=message):
105
+ Settings.from_env(env)
106
+
107
+
108
+ @pytest.mark.parametrize(
109
+ ("env", "missing"),
110
+ [
111
+ ({"OPENAI_API_KEY": "present"}, "HF_TOKEN"),
112
+ ({"HF_TOKEN": "present"}, "OPENAI_API_KEY"),
113
+ ],
114
+ )
115
+ def test_collection_validation_names_missing_credential(
116
+ env: dict[str, str], missing: str
117
+ ) -> None:
118
+ settings = Settings.from_env(env)
119
+
120
+ with pytest.raises(ValueError, match=missing):
121
+ settings.validate_for_collection()
122
+
123
+
124
+ def test_search_mode_does_not_require_collection_credentials() -> None:
125
+ settings = Settings.from_env({"BSIDES_MODE": "search"})
126
+
127
+ settings.validate_for_collection()
tests/test_runtime.py ADDED
@@ -0,0 +1,874 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ from __future__ import annotations
2
+
3
+ import os
4
+ import pathlib
5
+ import re
6
+ import sqlite3
7
+ import time
8
+ from dataclasses import replace
9
+
10
+ import numpy as np
11
+ import pytest
12
+ from fastapi.testclient import TestClient
13
+
14
+
15
+ MODEL_DDL = """
16
+ CREATE TABLE models (
17
+ row INTEGER PRIMARY KEY, model_id TEXT NOT NULL UNIQUE, author TEXT,
18
+ month TEXT, created_at TEXT, downloads INTEGER, likes INTEGER, tags TEXT,
19
+ base_model TEXT, relation TEXT, license TEXT, params INTEGER,
20
+ model_type TEXT, architectures TEXT, context_len INTEGER,
21
+ num_experts INTEGER, code_imports TEXT, gated INTEGER, config TEXT
22
+ );
23
+ """
24
+
25
+ KERNEL_DDL = """
26
+ CREATE TABLE kernels (
27
+ row INTEGER PRIMARY KEY, repo_id TEXT NOT NULL UNIQUE, author TEXT,
28
+ created_at TEXT, month TEXT, last_modified TEXT, downloads INTEGER,
29
+ likes INTEGER, tags TEXT, languages TEXT, accelerators TEXT,
30
+ torch_versions TEXT, cuda_archs TEXT, cpu_archs TEXT,
31
+ operating_systems TEXT, dtypes TEXT, intrinsics TEXT, kernel_names TEXT,
32
+ torch_ops TEXT, build_variants TEXT, variant_count INTEGER,
33
+ has_build_dir INTEGER, source_files INTEGER, document TEXT
34
+ );
35
+ """
36
+
37
+ CONTAINER_DDL = """
38
+ CREATE TABLE containers (
39
+ row INTEGER PRIMARY KEY, repo_id TEXT NOT NULL, repo_type TEXT NOT NULL,
40
+ artifact_path TEXT NOT NULL, artifact_kind TEXT NOT NULL,
41
+ author TEXT, sha TEXT NOT NULL, created_at TEXT, last_modified TEXT,
42
+ downloads INTEGER, likes INTEGER, tags TEXT, description TEXT,
43
+ artifact_files TEXT, qualification_reasons TEXT, base_images TEXT,
44
+ accelerators TEXT, cuda_versions TEXT, rocm_versions TEXT, python_versions TEXT,
45
+ node_versions TEXT, operating_systems TEXT, ports TEXT, services TEXT,
46
+ package_managers TEXT, frameworks TEXT, entrypoints TEXT, commands TEXT,
47
+ has_compose INTEGER, has_devcontainer INTEGER, has_multistage INTEGER,
48
+ source_files INTEGER, source_bytes INTEGER, document TEXT, document_hash TEXT
49
+ );
50
+ """
51
+
52
+ FIXTURE_MODELS = [
53
+ (0, "indie/alpha", "indie", "2024-01", "2024-01-10T00:00:00Z", 10, 1,
54
+ '["llama","sft"]', None, "finetune", "mit", None, "llama", None, None,
55
+ None, None, 0),
56
+ (1, "indie/beta", "lab", "2025-02", "2025-02-02T00:00:00Z", 500, 9,
57
+ '["mamba"]', None, None, "apache-2.0", None, "mamba", None, None, None,
58
+ None, 0),
59
+ ]
60
+
61
+ FIXTURE_KERNELS = [
62
+ (0, "alice/fused-gemm", "alice", "2025-01-10T00:00:00Z", "2025-01",
63
+ "2025-01-11T00:00:00Z", 100, 3, '["kernels"]', '["cuda","python"]',
64
+ '["nvidia"]', '["2.5"]', '["sm_90"]', '[]', '["linux"]',
65
+ '["bf16"]', '["mma_sync"]', '["fused_gemm"]', '["matmul"]',
66
+ '["torch25-cu124"]', 3, 1, 4, "fused gemm kernel"),
67
+ (1, "bob/paged-attention", "bob", "2024-06-02T00:00:00Z", "2024-06",
68
+ "2024-06-03T00:00:00Z", 20, 8, '["kernels"]', '["triton","python"]',
69
+ '["nvidia"]', '["2.4"]', '["sm_80"]', '[]', '["linux"]',
70
+ '["fp32"]', '["tl.dot"]', '["paged_attention"]', '["attention"]',
71
+ '[]', 1, 0, 2, "paged attention kernel"),
72
+ ]
73
+
74
+ FIXTURE_CONTAINERS = [
75
+ (0, "alice/cuda-recipe", "model", "Dockerfile", "recipe", "alice",
76
+ "sha-cuda", "2025-02-01T00:00:00Z", "2025-02-02T00:00:00Z", 300, 9,
77
+ '["docker","cuda"]', "CUDA PyTorch recipe", '["Dockerfile"]',
78
+ '["dockerfile","dependencies"]',
79
+ '["nvidia/cuda:12.8.0-runtime-ubuntu22.04"]', '["nvidia"]', '["12.8"]',
80
+ '[]', '["3.11"]', '[]', '["ubuntu22.04"]', '["7860"]', '["api"]',
81
+ '["pip"]', '["pytorch"]', '["python app.py"]', '["uvicorn app:app"]',
82
+ 0, 0, 1, 6, 4096, "python 3.11 cuda 12.8 pytorch recipe", "hash-cuda"),
83
+ (1, "bob/rocm-recipe", "space", "docker/Dockerfile", "recipe", "bob",
84
+ "sha-rocm", "2025-03-01T00:00:00Z", "2025-03-02T00:00:00Z", 120, 4,
85
+ '["docker","rocm"]', "ROCm build", '["docker/Dockerfile"]',
86
+ '["dockerfile"]', '["rocm/pytorch:latest"]', '["amd"]', '[]', '["6.1"]',
87
+ '["3.12"]', '["20"]', '["ubuntu24.04"]', '["7860"]', '[]',
88
+ '["apt","pip"]', '["pytorch"]', '[]', '["python launch.py"]',
89
+ 0, 0, 0, 5, 3072, "python 3.12 rocm pytorch recipe", "hash-rocm"),
90
+ (2, "carol/compose-stack", "dataset", "compose/Dockerfile", "recipe",
91
+ "carol", "sha-compose", "2025-04-01T00:00:00Z",
92
+ "2025-04-02T00:00:00Z", 80, 6, '["docker","compose"]',
93
+ "Compose-backed stack",
94
+ '["compose/Dockerfile","compose/docker-compose.yml"]',
95
+ '["compose","devcontainer"]', '["python:3.10-slim"]', '[]', '[]', '[]',
96
+ '["3.10"]', '["18"]', '["debian"]', '["5432","8888"]',
97
+ '["jupyter","postgres"]', '["apt","pip"]', '["fastapi"]',
98
+ '["./boot.sh"]', '["docker compose up"]', 1, 1, 0, 8, 8192,
99
+ "compose jupyter postgres fastapi stack", "hash-compose"),
100
+ (3, "dana/archive-bundle", "dataset", "artifacts/container.tar.gz",
101
+ "archive", "dana", "sha-archive", "2025-05-01T00:00:00Z",
102
+ "2025-05-02T00:00:00Z", 40, 1, '["docker","archive"]',
103
+ "Archived CUDA image", '["artifacts/container.tar.gz"]', '["archive"]',
104
+ '["ubuntu:22.04"]', '["nvidia"]', '["12.8"]', '[]', '["3.11"]', '[]',
105
+ '["ubuntu22.04"]', '["8080"]', '["worker"]', '["pip"]',
106
+ '["pytorch"]', '["./serve.sh"]', '["python worker.py"]',
107
+ 0, 0, 0, 2, 2048, "python 3.11 cuda 12.8 archive image",
108
+ "hash-archive"),
109
+ ]
110
+
111
+
112
+ def write_kernel_index(path, rows=FIXTURE_KERNELS, dimensions=4):
113
+ path.mkdir(parents=True, exist_ok=True)
114
+ connection = sqlite3.connect(path / "meta.sqlite")
115
+ connection.executescript(KERNEL_DDL)
116
+ connection.executemany(
117
+ f"INSERT INTO kernels VALUES ({','.join('?' * 24)})", rows)
118
+ connection.commit()
119
+ connection.close()
120
+ matrix = np.zeros((len(rows), dimensions), dtype=np.float16)
121
+ for row in range(len(rows)):
122
+ matrix[row, row % dimensions] = 1.0
123
+ np.save(path / "embeddings.f16.npy", matrix)
124
+
125
+
126
+ def write_container_index(path, rows=FIXTURE_CONTAINERS, dimensions=4):
127
+ path.mkdir(parents=True, exist_ok=True)
128
+ connection = sqlite3.connect(path / "meta.sqlite")
129
+ connection.executescript(CONTAINER_DDL)
130
+ connection.executemany(
131
+ f"INSERT INTO containers VALUES ({','.join('?' * 35)})", rows)
132
+ connection.commit()
133
+ connection.close()
134
+ matrix = np.zeros((len(rows), dimensions), dtype=np.float16)
135
+ for row in range(len(rows)):
136
+ matrix[row, row % dimensions] = 1.0
137
+ np.save(path / "embeddings.f16.npy", matrix)
138
+
139
+
140
+ @pytest.fixture
141
+ def http_client(tmp_path):
142
+ """The real app over a real two-row index, swapped in by registry key.
143
+
144
+ TestClient is built without its context manager on purpose: entering the
145
+ lifespan would start the collector thread, which this test is not about.
146
+ """
147
+ import app as app_module
148
+
149
+ from bsides.search_indexes import ModelAdapter
150
+
151
+ conn = sqlite3.connect(tmp_path / "meta.sqlite")
152
+ conn.executescript(MODEL_DDL)
153
+ conn.executemany(
154
+ f"INSERT INTO models VALUES ({','.join('?' * 19)})",
155
+ [row + (None,) for row in FIXTURE_MODELS])
156
+ conn.commit()
157
+ conn.close()
158
+ matrix = np.zeros((2, 4), dtype=np.float16)
159
+ matrix[0, 0] = matrix[1, 1] = 1.0
160
+ np.save(tmp_path / "embeddings.f16.npy", matrix)
161
+
162
+ original = app_module.registry.index("models")
163
+ spec = replace(app_module.MODEL_SPEC, local_dir=tmp_path, dimensions=4)
164
+ index = app_module.registry.register(ModelAdapter(spec))
165
+ index.load()
166
+ assert index.ready.is_set(), index.error
167
+ try:
168
+ yield TestClient(app_module.app)
169
+ finally:
170
+ app_module.registry.register(original.adapter)
171
+
172
+
173
+ @pytest.fixture
174
+ def kernel_http_client(http_client, tmp_path):
175
+ import app as app_module
176
+
177
+ from bsides.search_indexes import KernelAdapter
178
+
179
+ kernel_dir = tmp_path / "kernels"
180
+ write_kernel_index(kernel_dir)
181
+ original = app_module.registry.index("kernels")
182
+ spec = replace(app_module.KERNEL_SPEC, local_dir=kernel_dir, dimensions=4)
183
+ index = app_module.registry.register(KernelAdapter(spec))
184
+ index.load()
185
+ assert index.ready.is_set(), index.error
186
+ try:
187
+ yield http_client
188
+ finally:
189
+ app_module.registry.register(original.adapter)
190
+
191
+
192
+ @pytest.fixture
193
+ def docker_http_client(http_client, tmp_path):
194
+ import app as app_module
195
+
196
+ from bsides.search_indexes import ContainerAdapter
197
+
198
+ docker_dir = tmp_path / "docker"
199
+ write_container_index(docker_dir)
200
+ original = app_module.registry.index("docker")
201
+ spec = replace(app_module.DOCKER_SPEC, local_dir=docker_dir, dimensions=4)
202
+ index = app_module.registry.register(ContainerAdapter(spec))
203
+ index.load()
204
+ assert index.ready.is_set(), index.error
205
+ try:
206
+ yield http_client
207
+ finally:
208
+ app_module.registry.register(original.adapter)
209
+
210
+
211
+ @pytest.fixture
212
+ def all_corpora_http_client(kernel_http_client, tmp_path):
213
+ import app as app_module
214
+
215
+ from bsides.search_indexes import ContainerAdapter
216
+
217
+ docker_dir = tmp_path / "docker"
218
+ write_container_index(docker_dir)
219
+ original = app_module.registry.index("docker")
220
+ spec = replace(app_module.DOCKER_SPEC, local_dir=docker_dir, dimensions=4)
221
+ index = app_module.registry.register(ContainerAdapter(spec))
222
+ index.load()
223
+ assert index.ready.is_set(), index.error
224
+ try:
225
+ yield kernel_http_client
226
+ finally:
227
+ app_module.registry.register(original.adapter)
228
+
229
+
230
+ def test_app_registers_kernel_corpus_configuration() -> None:
231
+ import app as app_module
232
+
233
+ spec = app_module.registry.index("kernels").spec
234
+
235
+ assert spec is app_module.KERNEL_SPEC
236
+ assert spec.key == "kernels"
237
+ assert spec.dataset_id == os.environ.get(
238
+ "BSIDES_KERNEL_DATASET", "juiceb0xc0de/b-sides-v2-kernels")
239
+ assert spec.local_dir == app_module.INDEX_DIR / "kernels"
240
+ assert spec.table == "kernels"
241
+ assert spec.id_column == "repo_id"
242
+ assert spec.embed_model == "text-embedding-3-small"
243
+ assert spec.dimensions == 256
244
+
245
+
246
+ def test_app_registers_docker_corpus_configuration() -> None:
247
+ import app as app_module
248
+
249
+ spec = app_module.registry.index("docker").spec
250
+
251
+ assert spec is app_module.DOCKER_SPEC
252
+ assert spec.key == "docker"
253
+ assert spec.dataset_id == os.environ.get(
254
+ "BSIDES_CONTAINER_DATASET", "juiceb0xc0de/b-sides-v2-containers")
255
+ assert spec.local_dir == app_module.INDEX_DIR / "docker"
256
+ assert spec.table == "containers"
257
+ assert spec.id_column == "repo_id"
258
+ assert spec.embed_model == "text-embedding-3-small"
259
+ assert spec.dimensions == 256
260
+
261
+
262
+ def test_kernel_facets_and_search_use_kernel_response_shape(
263
+ kernel_http_client) -> None:
264
+ facets = kernel_http_client.get("/api/facets?corpus=kernels")
265
+ search = kernel_http_client.post(
266
+ "/api/search?corpus=kernels",
267
+ json={"sort": "downloads", "filters": {
268
+ "languages": ["cuda", "triton"],
269
+ "has_build_dir": True,
270
+ "variant_count_min": 2,
271
+ }},
272
+ )
273
+
274
+ assert facets.status_code == 200
275
+ assert facets.json()["total"] == 2
276
+ assert {item["v"]: item["n"]
277
+ for item in facets.json()["languages"]} == {
278
+ "python": 2, "cuda": 1, "triton": 1}
279
+ assert search.status_code == 200
280
+ assert set(search.json()) == {"total", "kernels"}
281
+ assert search.json()["total"] == 1
282
+ assert [item["repo_id"] for item in search.json()["kernels"]] == [
283
+ "alice/fused-gemm"]
284
+ assert search.json()["kernels"][0]["languages"] == ["cuda", "python"]
285
+
286
+
287
+ def test_docker_facets_and_search_use_container_response_shape(
288
+ docker_http_client) -> None:
289
+ facets = docker_http_client.get("/api/facets?corpus=docker")
290
+ search = docker_http_client.post(
291
+ "/api/search?corpus=docker",
292
+ json={"sort": "downloads", "filters": {
293
+ "artifact_kinds": ["recipe"],
294
+ "python_versions": ["3.11"],
295
+ "cuda_versions": ["12.8"],
296
+ "frameworks": ["pytorch"],
297
+ }},
298
+ )
299
+
300
+ assert facets.status_code == 200
301
+ assert facets.json()["total"] == 4
302
+ assert {item["v"]: item["n"]
303
+ for item in facets.json()["artifact_kinds"]} == {
304
+ "recipe": 3, "archive": 1}
305
+ assert search.status_code == 200
306
+ assert set(search.json()) == {"total", "containers"}
307
+ assert search.json()["total"] == 1
308
+ assert [item["repo_id"] for item in search.json()["containers"]] == [
309
+ "alice/cuda-recipe"]
310
+ assert search.json()["containers"][0]["base_images"] == [
311
+ "nvidia/cuda:12.8.0-runtime-ubuntu22.04"]
312
+
313
+
314
+ def test_failed_kernel_load_does_not_affect_model_search(
315
+ http_client, tmp_path) -> None:
316
+ import app as app_module
317
+
318
+ from bsides.search_indexes import KernelAdapter
319
+
320
+ bad_dir = tmp_path / "bad-kernels"
321
+ write_kernel_index(bad_dir, rows=FIXTURE_KERNELS[:1], dimensions=8)
322
+ original = app_module.registry.index("kernels")
323
+ spec = replace(app_module.KERNEL_SPEC, local_dir=bad_dir, dimensions=4)
324
+ failed = app_module.registry.register(KernelAdapter(spec))
325
+ failed.load()
326
+ try:
327
+ kernel_response = http_client.get("/api/facets?corpus=kernels")
328
+ model_response = http_client.post(
329
+ "/api/search", json={"sort": "downloads"})
330
+
331
+ assert failed.error and "dimensions" in failed.error
332
+ assert kernel_response.status_code == 503
333
+ assert model_response.status_code == 200
334
+ assert model_response.json()["total"] == 2
335
+ assert set(model_response.json()) == {"total", "models"}
336
+ finally:
337
+ app_module.registry.register(original.adapter)
338
+
339
+
340
+ def test_failed_docker_load_does_not_affect_model_search(
341
+ http_client, tmp_path) -> None:
342
+ import app as app_module
343
+
344
+ from bsides.search_indexes import ContainerAdapter
345
+
346
+ bad_dir = tmp_path / "bad-docker"
347
+ write_container_index(bad_dir, rows=FIXTURE_CONTAINERS[:1], dimensions=8)
348
+ original = app_module.registry.index("docker")
349
+ spec = replace(app_module.DOCKER_SPEC, local_dir=bad_dir, dimensions=4)
350
+ failed = app_module.registry.register(ContainerAdapter(spec))
351
+ failed.load()
352
+ try:
353
+ docker_response = http_client.get("/api/facets?corpus=docker")
354
+ model_response = http_client.post(
355
+ "/api/search", json={"sort": "downloads"})
356
+
357
+ assert failed.error and "dimensions" in failed.error
358
+ assert docker_response.status_code == 503
359
+ assert model_response.status_code == 200
360
+ assert model_response.json()["total"] == 2
361
+ assert set(model_response.json()) == {"total", "models"}
362
+ finally:
363
+ app_module.registry.register(original.adapter)
364
+
365
+
366
+ def test_reload_endpoint_hot_swaps_only_the_kernel_corpus(
367
+ kernel_http_client, monkeypatch) -> None:
368
+ import app as app_module
369
+
370
+ index = app_module.registry.index("kernels")
371
+ target = index.spec.local_dir
372
+ replacement = [(0, *FIXTURE_KERNELS[1][1:])]
373
+
374
+ def downloader(dataset_id, **kwargs):
375
+ assert dataset_id == index.spec.dataset_id
376
+ assert kwargs["local_dir"] == target
377
+ (target / "meta.sqlite").unlink(missing_ok=True)
378
+ (target / "embeddings.f16.npy").unlink(missing_ok=True)
379
+ write_kernel_index(target, rows=replacement, dimensions=4)
380
+
381
+ monkeypatch.setattr(app_module, "RELOAD_TOKEN", "correct-horse")
382
+ monkeypatch.setattr(app_module.registry, "downloader", downloader)
383
+
384
+ response = kernel_http_client.post(
385
+ "/api/reload?corpus=kernels", json={"token": "correct-horse"})
386
+
387
+ assert response.status_code == 200
388
+ assert response.json() == {
389
+ "status": "reload started", "corpus": "kernels"}
390
+ assert _await_rows(index, 1), index.error
391
+ kernel_body = kernel_http_client.post(
392
+ "/api/search?corpus=kernels", json={"sort": "downloads"}).json()
393
+ model_body = kernel_http_client.post(
394
+ "/api/search", json={"sort": "downloads"}).json()
395
+ assert [item["repo_id"] for item in kernel_body["kernels"]] == [
396
+ "bob/paged-attention"]
397
+ assert model_body["total"] == 2
398
+ assert set(model_body) == {"total", "models"}
399
+
400
+
401
+ def test_reload_endpoint_hot_swaps_only_the_docker_corpus(
402
+ docker_http_client, monkeypatch) -> None:
403
+ import app as app_module
404
+
405
+ index = app_module.registry.index("docker")
406
+ target = index.spec.local_dir
407
+ replacement = [(0, *FIXTURE_CONTAINERS[1][1:])]
408
+
409
+ def downloader(dataset_id, **kwargs):
410
+ assert dataset_id == index.spec.dataset_id
411
+ assert kwargs["local_dir"] == target
412
+ (target / "meta.sqlite").unlink(missing_ok=True)
413
+ (target / "embeddings.f16.npy").unlink(missing_ok=True)
414
+ write_container_index(target, rows=replacement, dimensions=4)
415
+
416
+ monkeypatch.setattr(app_module, "RELOAD_TOKEN", "correct-horse")
417
+ monkeypatch.setattr(app_module.registry, "downloader", downloader)
418
+
419
+ response = docker_http_client.post(
420
+ "/api/reload?corpus=docker", json={"token": "correct-horse"})
421
+
422
+ assert response.status_code == 200
423
+ assert response.json() == {
424
+ "status": "reload started", "corpus": "docker"}
425
+ assert _await_rows(index, 1), index.error
426
+ docker_body = docker_http_client.post(
427
+ "/api/search?corpus=docker", json={"sort": "downloads"}).json()
428
+ model_body = docker_http_client.post(
429
+ "/api/search", json={"sort": "downloads"}).json()
430
+ assert [item["repo_id"] for item in docker_body["containers"]] == [
431
+ "bob/rocm-recipe"]
432
+ assert model_body["total"] == 2
433
+ assert set(model_body) == {"total", "models"}
434
+
435
+
436
+ def test_search_response_keeps_the_total_models_envelope(http_client) -> None:
437
+ response = http_client.post("/api/search", json={"sort": "downloads"})
438
+
439
+ assert response.status_code == 200
440
+ body = response.json()
441
+ assert set(body) == {"total", "models"}
442
+ assert body["total"] == 2
443
+ assert [m["model_id"] for m in body["models"]] == [
444
+ "indie/beta", "indie/alpha"]
445
+ assert body["models"][0]["tags"] == ["mamba"] # JSON columns decoded
446
+
447
+
448
+ def test_model_search_exclusions_are_explicit_and_api_compatible(
449
+ http_client) -> None:
450
+ default = http_client.post(
451
+ "/api/search", json={"sort": "downloads"}).json()
452
+ excluded = http_client.post(
453
+ "/api/search",
454
+ json={"sort": "downloads", "exclude_families": ["llama"]},
455
+ )
456
+
457
+ assert excluded.status_code == 200
458
+ assert set(excluded.json()) == {"total", "models"}
459
+ assert {m["model_id"] for m in default["models"]} == {
460
+ "indie/alpha", "indie/beta"}
461
+ assert [m["model_id"] for m in excluded.json()["models"]] == [
462
+ "indie/beta"]
463
+
464
+
465
+ def test_search_defaults_to_the_models_corpus(http_client) -> None:
466
+ default = http_client.post("/api/search", json={"sort": "downloads"})
467
+ explicit = http_client.post(
468
+ "/api/search?corpus=models", json={"sort": "downloads"})
469
+
470
+ assert default.json() == explicit.json()
471
+
472
+
473
+ class RecordingClient:
474
+ """Embedding client double that records the app's outbound request."""
475
+
476
+ def __init__(self) -> None:
477
+ self.calls: list[dict] = []
478
+ self.embeddings = self
479
+
480
+ def create(self, **kwargs):
481
+ self.calls.append(kwargs)
482
+ return type("Response", (), {
483
+ "data": [type("Item", (), {"embedding": [1.0, 0.0, 0.0, 0.0]})()]
484
+ })()
485
+
486
+
487
+ def test_semantic_searches_use_their_assigned_corpus_clients(
488
+ all_corpora_http_client, monkeypatch) -> None:
489
+ import app as app_module
490
+
491
+ model = RecordingClient()
492
+ kernel = RecordingClient()
493
+ docker = RecordingClient()
494
+ monkeypatch.setattr(app_module, "query_clients", {
495
+ "models": model, "kernels": kernel, "docker": docker,
496
+ })
497
+
498
+ model_response = all_corpora_http_client.post(
499
+ "/api/search", json={"query": "mamba hybrid architecture"})
500
+ kernel_response = all_corpora_http_client.post(
501
+ "/api/search?corpus=kernels", json={"query": "fused attention"})
502
+ docker_response = all_corpora_http_client.post(
503
+ "/api/search?corpus=docker", json={"query": "python 3.11 cuda 12.8"})
504
+
505
+ assert model_response.status_code == 200
506
+ assert kernel_response.status_code == 200
507
+ assert docker_response.status_code == 200
508
+ assert model.calls == [{
509
+ "model": "text-embedding-3-small",
510
+ "dimensions": 4,
511
+ "input": "mamba hybrid architecture",
512
+ }]
513
+ assert kernel.calls == [{
514
+ "model": "text-embedding-3-small",
515
+ "dimensions": 4,
516
+ "input": "fused attention",
517
+ }]
518
+ assert docker.calls == [{
519
+ "model": "text-embedding-3-small",
520
+ "dimensions": 4,
521
+ "input": "python 3.11 cuda 12.8",
522
+ }]
523
+
524
+
525
+ def test_models_client_remains_the_oai_compatibility_shim() -> None:
526
+ import app as app_module
527
+
528
+ assert set(app_module.query_clients) == {"models", "kernels", "docker"}
529
+ assert app_module.oai is app_module.query_clients["models"]
530
+
531
+
532
+ def test_semantic_search_without_a_key_is_unavailable_not_a_crash(
533
+ http_client, monkeypatch) -> None:
534
+ import app as app_module
535
+
536
+ monkeypatch.setitem(app_module.query_clients, "models", None)
537
+ response = http_client.post(
538
+ "/api/search", json={"query": "mamba hybrid architecture"})
539
+
540
+ assert response.status_code == 503
541
+ assert "OPENAI_API_KEY" in response.json()["detail"]
542
+
543
+
544
+ def test_docker_semantic_search_without_a_key_is_unavailable_not_a_crash(
545
+ docker_http_client, monkeypatch) -> None:
546
+ import app as app_module
547
+
548
+ monkeypatch.setitem(app_module.query_clients, "docker", None)
549
+ response = docker_http_client.post(
550
+ "/api/search?corpus=docker", json={"query": "python 3.11 cuda 12.8"})
551
+ model_response = docker_http_client.post(
552
+ "/api/search", json={"sort": "downloads"})
553
+
554
+ assert response.status_code == 503
555
+ assert "OPENAI_DOCKER_API_KEY" in response.json()["detail"]
556
+ assert model_response.status_code == 200
557
+
558
+
559
+ def test_facets_and_picks_still_answer(http_client) -> None:
560
+ facets = http_client.get("/api/facets")
561
+ assert facets.status_code == 200
562
+ assert facets.json()["total"] == 2
563
+ assert {a["v"] for a in facets.json()["archs"]} == {"llama", "mamba"}
564
+
565
+ picks = http_client.get("/api/picks")
566
+ assert picks.status_code == 200
567
+ assert "picks" in picks.json()
568
+
569
+
570
+ def test_unknown_corpus_is_a_404(http_client) -> None:
571
+ assert http_client.get("/api/facets?corpus=bogus").status_code == 404
572
+
573
+
574
+ # ── reload: the publish_loop.py wire contract ─────────────────────────────────
575
+ # publish_loop.reload_space POSTs {"token": ...} and reads 200 / 403 / 503.
576
+
577
+ def _await_rows(index, expected, timeout=10.0):
578
+ deadline = time.monotonic() + timeout
579
+ while time.monotonic() < deadline:
580
+ if index.rows == expected and index.ready.is_set():
581
+ return True
582
+ time.sleep(0.01)
583
+ return False
584
+
585
+
586
+ def test_reload_without_a_configured_token_is_503(http_client, monkeypatch):
587
+ import app as app_module
588
+
589
+ monkeypatch.setattr(app_module, "RELOAD_TOKEN", "")
590
+ response = http_client.post("/api/reload", json={"token": "anything"})
591
+
592
+ assert response.status_code == 503
593
+ assert "BSIDES_RELOAD_TOKEN" in response.json()["detail"]
594
+
595
+
596
+ def test_reload_with_a_wrong_token_is_403(http_client, monkeypatch):
597
+ import app as app_module
598
+
599
+ monkeypatch.setattr(app_module, "RELOAD_TOKEN", "correct-horse")
600
+ index = app_module.registry.index("models")
601
+
602
+ response = http_client.post("/api/reload", json={"token": "guess"})
603
+
604
+ assert response.status_code == 403
605
+ assert index.rows == 2 # untouched
606
+
607
+
608
+ def test_reload_with_an_empty_body_is_rejected(http_client, monkeypatch):
609
+ import app as app_module
610
+
611
+ monkeypatch.setattr(app_module, "RELOAD_TOKEN", "correct-horse")
612
+ assert http_client.post("/api/reload", json={}).status_code == 403
613
+
614
+
615
+ def test_reload_with_the_right_token_swaps_the_index(http_client, monkeypatch):
616
+ import app as app_module
617
+
618
+ index = app_module.registry.index("models")
619
+ target = index.spec.local_dir
620
+
621
+ def downloader(dataset_id, **kwargs):
622
+ (target / "meta.sqlite").unlink(missing_ok=True)
623
+ (target / "embeddings.f16.npy").unlink(missing_ok=True)
624
+ conn = sqlite3.connect(target / "meta.sqlite")
625
+ conn.executescript(MODEL_DDL)
626
+ conn.executemany(
627
+ f"INSERT INTO models VALUES ({','.join('?' * 19)})",
628
+ [FIXTURE_MODELS[0] + (None,)])
629
+ conn.commit()
630
+ conn.close()
631
+ np.save(target / "embeddings.f16.npy",
632
+ np.array([[1, 0, 0, 0]], dtype=np.float16))
633
+
634
+ monkeypatch.setattr(app_module, "RELOAD_TOKEN", "correct-horse")
635
+ monkeypatch.setattr(app_module.registry, "downloader", downloader)
636
+
637
+ response = http_client.post("/api/reload", json={"token": "correct-horse"})
638
+
639
+ assert response.status_code == 200
640
+ assert response.json()["status"] == "reload started"
641
+ assert _await_rows(index, 1), index.error
642
+ assert index.error is None
643
+ # the swap is visible over HTTP, not just in memory
644
+ body = http_client.post("/api/search", json={"sort": "downloads"}).json()
645
+ assert body["total"] == 1
646
+ assert [m["model_id"] for m in body["models"]] == ["indie/alpha"]
647
+
648
+
649
+ def test_reload_of_an_unknown_corpus_is_a_404(http_client, monkeypatch):
650
+ import app as app_module
651
+
652
+ monkeypatch.setattr(app_module, "RELOAD_TOKEN", "correct-horse")
653
+ response = http_client.post(
654
+ "/api/reload?corpus=bogus", json={"token": "correct-horse"})
655
+
656
+ assert response.status_code == 404
657
+ assert not app_module._reload_lock.locked() # gate ran before the lock
658
+
659
+
660
+ def test_endpoints_503_until_the_index_is_ready(tmp_path) -> None:
661
+ import app as app_module
662
+
663
+ from bsides.search_indexes import ModelAdapter
664
+
665
+ original = app_module.registry.index("models")
666
+ spec = replace(app_module.MODEL_SPEC, local_dir=tmp_path)
667
+ app_module.registry.register(ModelAdapter(spec)) # registered, never loaded
668
+ try:
669
+ client = TestClient(app_module.app)
670
+ assert client.post("/api/search", json={}).status_code == 503
671
+ assert client.get("/api/facets").status_code == 503
672
+ finally:
673
+ app_module.registry.register(original.adapter)
674
+
675
+
676
+ def test_search_index_download_excludes_crawler_staging() -> None:
677
+ # The index dataset also carries the multi-GB crawler checkpoints under
678
+ # staging/. The models spec must allow-list only the two index files, so a
679
+ # boot download can never drag the staging tree onto the Space.
680
+ import app as app_module
681
+
682
+ from bsides.search_indexes import download_index
683
+
684
+ calls: list[tuple[str, dict]] = []
685
+
686
+ def downloader(dataset_id: str, **kwargs) -> None:
687
+ calls.append((dataset_id, kwargs))
688
+
689
+ spec = app_module.MODEL_SPEC
690
+ download_index(downloader, spec)
691
+
692
+ assert calls == [
693
+ (
694
+ spec.dataset_id,
695
+ {
696
+ "repo_type": "dataset",
697
+ "local_dir": spec.local_dir,
698
+ "allow_patterns": ["embeddings.f16.npy", "meta.sqlite"],
699
+ },
700
+ )
701
+ ]
702
+
703
+
704
+ def test_docker_index_download_allows_only_published_search_files() -> None:
705
+ import app as app_module
706
+
707
+ from bsides.search_indexes import download_index
708
+
709
+ calls: list[tuple[str, dict]] = []
710
+
711
+ def downloader(dataset_id: str, **kwargs) -> None:
712
+ calls.append((dataset_id, kwargs))
713
+
714
+ spec = app_module.DOCKER_SPEC
715
+ download_index(downloader, spec)
716
+
717
+ assert calls == [
718
+ (
719
+ spec.dataset_id,
720
+ {
721
+ "repo_type": "dataset",
722
+ "local_dir": spec.local_dir,
723
+ "allow_patterns": ["embeddings.f16.npy", "meta.sqlite"],
724
+ },
725
+ )
726
+ ]
727
+
728
+
729
+ def test_search_query_uses_the_loaded_index_dimensions(tmp_path) -> None:
730
+ # Dimensions sent to the embedding API come off the loaded matrix, not the
731
+ # spec, so a corpus published at a different width still queries correctly.
732
+ from bsides.search_indexes import LoadedIndex, embed_query
733
+
734
+ import app as app_module
735
+
736
+ calls: list[dict] = []
737
+
738
+ class Embeddings:
739
+ def create(self, **kwargs):
740
+ calls.append(kwargs)
741
+ item = type("Item", (), {"embedding": [1.0] + [0.0] * 255})()
742
+ return type("Response", (), {"data": [item]})()
743
+
744
+ index = LoadedIndex(replace(app_module.MODEL_SPEC, local_dir=tmp_path))
745
+ index.matrix = np.zeros((1, 256), dtype=np.float32)
746
+ client = type("Client", (), {"embeddings": Embeddings()})()
747
+
748
+ vector = embed_query(index, client, "odd attention")
749
+
750
+ assert vector.shape == (256,)
751
+ assert calls == [{
752
+ "model": "text-embedding-3-small",
753
+ "dimensions": 256,
754
+ "input": "odd attention",
755
+ }]
756
+
757
+
758
+ # ── three-corpus frontend contract ───────────────────────────────────────────
759
+ # The UI is one hand-written file with no JavaScript test runner, so these read
760
+ # the source and pin the product decisions that regress silently: the corpus
761
+ # list, per-corpus state, the visible default exclusions, and the generic URL
762
+ # builder that a models-vs-kernels ternary would quietly break.
763
+
764
+ DEFAULT_EXCLUDED_FAMILIES = (
765
+ 'llama', 'qwen', 'gemma', 'mistral', 'deepseek', 'phi', 'granite')
766
+
767
+
768
+ def _page() -> str:
769
+ path = pathlib.Path(__file__).resolve().parents[1] / 'static' / 'index.html'
770
+ return path.read_text(encoding='utf-8')
771
+
772
+
773
+ def _function_source(page: str, name: str) -> str:
774
+ body = page.split(f'function {name}(', 1)[1]
775
+ return body.split('\n}\n', 1)[0]
776
+
777
+
778
+ def test_frontend_offers_all_three_corpora() -> None:
779
+ page = _page()
780
+
781
+ assert '<option value="models" selected>Models</option>' in page
782
+ assert '<option value="kernels">Kernels</option>' in page
783
+ assert '<option value="docker">Docker Images</option>' in page
784
+
785
+
786
+ def test_frontend_tracks_docker_in_every_corpus_state_map() -> None:
787
+ page = _page()
788
+
789
+ assert 'facets: { models: null, kernels: null, docker: null }' in page
790
+ assert 'ready: { models: false, kernels: false, docker: false }' in page
791
+ assert 'loading: { models: null, kernels: null, docker: null }' in page
792
+ assert 'docker: {' in page.split('sel: {', 1)[1]
793
+
794
+
795
+ def test_frontend_checks_model_exclusions_by_default() -> None:
796
+ page = _page()
797
+
798
+ declaration = re.search(
799
+ r'const DEFAULT_EXCLUDED_FAMILIES = \[(.*?)\];', page, re.S)
800
+ assert declaration is not None
801
+ assert re.findall(r"'([a-z]+)'", declaration.group(1)) == list(
802
+ DEFAULT_EXCLUDED_FAMILIES)
803
+
804
+ # Checked means hidden, and both exclusions start on for the browser only.
805
+ assert 'excludedFamilies: new Set(DEFAULT_EXCLUDED_FAMILIES)' in page
806
+ assert 'excludeQuantizations: true' in page
807
+ assert 'exclude_families: [...state.sel.models.excludedFamilies]' in page
808
+ assert ('exclude_quantizations: state.sel.models.excludeQuantizations'
809
+ in page)
810
+
811
+
812
+ def test_frontend_restores_default_exclusions_when_clearing_the_deck() -> None:
813
+ page = _page()
814
+ source = _function_source(page, 'clearModels')
815
+
816
+ # `clear the deck` restores the product defaults; it must not switch the
817
+ # exclusions off, which would silently widen every subsequent search.
818
+ assert 'DEFAULT_EXCLUDED_FAMILIES' in source
819
+ assert 'excludeQuantizations = true' in source
820
+
821
+
822
+ def test_frontend_keeps_model_size_as_precise_billion_boxes() -> None:
823
+ page = _page()
824
+
825
+ assert 'id="f-pmin"' in page and 'id="f-pmax"' in page
826
+ assert page.count('step="0.1"') == 2
827
+ assert 'Model size (billions)' in page
828
+ assert '1.5B–3.5B' in page
829
+ assert '* 1e9' in page
830
+ # An inverted range is flagged rather than sent as an impossible request.
831
+ assert 'id="size-warn"' in page
832
+
833
+
834
+ def test_frontend_defaults_docker_to_recipes_with_an_archive_toggle() -> None:
835
+ page = _page()
836
+ source = _function_source(page, 'buildDockerReq')
837
+
838
+ assert 'includeArchives: false' in page
839
+ assert 'id="d-archives"' in page
840
+ assert "filters.artifact_kinds = ['recipe']" in source
841
+ # Enabling archives removes the constraint instead of adding a second kind.
842
+ assert 'if (!s.includeArchives)' in source
843
+ assert "'archive'" not in source
844
+
845
+
846
+ def test_frontend_routes_every_corpus_through_one_url_builder() -> None:
847
+ page = _page()
848
+
849
+ assert ("const resultKey = {models:'models', kernels:'kernels', "
850
+ "docker:'containers'}" in page)
851
+ assert 'const endpoint = (kind, corpus) =>' in page
852
+ # No binary ternary may survive, or Docker silently routes to kernels.
853
+ assert "'/api/search?corpus=kernels'" not in page
854
+ assert "'/api/facets?corpus=kernels'" not in page
855
+
856
+
857
+ def test_frontend_escapes_every_docker_card_value() -> None:
858
+ page = _page()
859
+ source = _function_source(page, 'dockerCard')
860
+
861
+ assert 'esc(' in source
862
+ # No raw record field may reach the template unescaped.
863
+ assert re.search(r'\$\{c\.[A-Za-z_]+\}', source) is None
864
+
865
+
866
+ def test_frontend_links_docker_artifacts_to_valid_hugging_face_targets() -> None:
867
+ page = _page()
868
+ source = _function_source(page, 'containerHref')
869
+
870
+ assert "'spaces/'" in source and "'datasets/'" in source
871
+ assert 'blob/' in source
872
+ # Bucket rows and synthetic <recipe> paths have no blob URL; those fall
873
+ # back to the repository landing page instead of a 404.
874
+ assert '<recipe>' in source
tests/test_search_indexes.py ADDED
@@ -0,0 +1,1287 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """Tests for the reusable multi-corpus index core (bsides.search_indexes).
2
+
3
+ Covers the loader validation contract, the registry's failure isolation,
4
+ the ModelAdapter (today's model behavior, relocated), and the shared
5
+ ranking functions. Fixtures write real `meta.sqlite` + `embeddings.f16.npy`
6
+ pairs to tmp dirs so the loader is exercised exactly as on the Space.
7
+ """
8
+
9
+ from __future__ import annotations
10
+
11
+ import json
12
+ import sqlite3
13
+ from pathlib import Path
14
+ from types import SimpleNamespace
15
+
16
+ import numpy as np
17
+ import pytest
18
+
19
+ from bsides.search_indexes import (
20
+ MAX_QUERY_CHARS,
21
+ MATRIX_NAME,
22
+ SQLITE_NAME,
23
+ IndexRegistry,
24
+ IndexSpec,
25
+ LoadedIndex,
26
+ ModelAdapter,
27
+ SearchReq,
28
+ UnavailableError,
29
+ download_index,
30
+ embed_query,
31
+ semantic_rank,
32
+ structured_page,
33
+ )
34
+
35
+ # CARD_COLS + created_at (needed by the sort paths), matching the published
36
+ # models schema subset the app actually reads.
37
+ MODEL_DDL = """
38
+ CREATE TABLE models (
39
+ row INTEGER PRIMARY KEY,
40
+ model_id TEXT NOT NULL UNIQUE,
41
+ author TEXT,
42
+ month TEXT,
43
+ created_at TEXT,
44
+ downloads INTEGER,
45
+ likes INTEGER,
46
+ tags TEXT,
47
+ base_model TEXT,
48
+ relation TEXT,
49
+ license TEXT,
50
+ params INTEGER,
51
+ model_type TEXT,
52
+ architectures TEXT,
53
+ context_len INTEGER,
54
+ num_experts INTEGER,
55
+ code_imports TEXT,
56
+ gated INTEGER,
57
+ config TEXT
58
+ );
59
+ """
60
+
61
+ KERNEL_DDL = """
62
+ CREATE TABLE kernels (
63
+ row INTEGER PRIMARY KEY, repo_id TEXT NOT NULL UNIQUE, author TEXT,
64
+ created_at TEXT, month TEXT, last_modified TEXT, downloads INTEGER,
65
+ likes INTEGER, tags TEXT, languages TEXT, accelerators TEXT,
66
+ torch_versions TEXT, cuda_archs TEXT, cpu_archs TEXT,
67
+ operating_systems TEXT, dtypes TEXT, intrinsics TEXT, kernel_names TEXT,
68
+ torch_ops TEXT, build_variants TEXT, variant_count INTEGER,
69
+ has_build_dir INTEGER, source_files INTEGER, document TEXT
70
+ );
71
+ """
72
+
73
+ CONTAINER_DDL = """
74
+ CREATE TABLE containers (
75
+ row INTEGER PRIMARY KEY, repo_id TEXT NOT NULL, repo_type TEXT NOT NULL,
76
+ artifact_path TEXT NOT NULL, artifact_kind TEXT NOT NULL,
77
+ author TEXT, sha TEXT NOT NULL, created_at TEXT, last_modified TEXT,
78
+ downloads INTEGER, likes INTEGER, tags TEXT, description TEXT,
79
+ artifact_files TEXT, qualification_reasons TEXT, base_images TEXT,
80
+ accelerators TEXT, cuda_versions TEXT, rocm_versions TEXT, python_versions TEXT,
81
+ node_versions TEXT, operating_systems TEXT, ports TEXT, services TEXT,
82
+ package_managers TEXT, frameworks TEXT, entrypoints TEXT, commands TEXT,
83
+ has_compose INTEGER, has_devcontainer INTEGER, has_multistage INTEGER,
84
+ source_files INTEGER, source_bytes INTEGER, document TEXT, document_hash TEXT
85
+ );
86
+ """
87
+
88
+
89
+ def model_row(row, model_id, *, author='indie', month='2024-01',
90
+ created_at='2024-01-15T00:00:00Z', downloads=0, likes=0,
91
+ tags='[]', base_model=None, relation=None, license=None,
92
+ params=None, model_type=None, architectures=None,
93
+ context_len=None, num_experts=None, code_imports=None,
94
+ gated=0, config=None):
95
+ return (row, model_id, author, month, created_at, downloads, likes, tags,
96
+ base_model, relation, license, params, model_type, architectures,
97
+ context_len, num_experts, code_imports, gated, config)
98
+
99
+
100
+ def kernel_row(row, repo_id, *, author='alice', month='2025-01',
101
+ created_at='2025-01-15T00:00:00Z', downloads=100, likes=2,
102
+ tags='["kernels"]', languages='["cuda","python"]',
103
+ accelerators='["nvidia"]', torch_versions='["2.5"]',
104
+ cuda_archs='["sm_90"]', cpu_archs='[]',
105
+ operating_systems='["linux"]', dtypes='["bf16","fp16"]',
106
+ intrinsics='["mma_sync","__syncthreads"]',
107
+ kernel_names='["fused_gemm"]', torch_ops='["matmul"]',
108
+ build_variants='["torch25-cu124"]', variant_count=3,
109
+ has_build_dir=1, source_files=4, document='fused gemm kernel'):
110
+ return (row, repo_id, author, created_at, month, created_at, downloads,
111
+ likes, tags, languages, accelerators, torch_versions, cuda_archs,
112
+ cpu_archs, operating_systems, dtypes, intrinsics, kernel_names,
113
+ torch_ops, build_variants, variant_count, has_build_dir,
114
+ source_files, document)
115
+
116
+
117
+ def _json_text(value):
118
+ return json.dumps(value)
119
+
120
+
121
+ def container_row(
122
+ row, repo_id, *, repo_type='model', artifact_path='Dockerfile',
123
+ artifact_kind='recipe', author='alice', sha='sha-1',
124
+ created_at='2025-01-15T00:00:00Z',
125
+ last_modified='2025-01-16T00:00:00Z', downloads=100, likes=2,
126
+ tags=None, description='container recipe', artifact_files=None,
127
+ qualification_reasons=None, base_images=None, accelerators=None,
128
+ cuda_versions=None, rocm_versions=None, python_versions=None,
129
+ node_versions=None, operating_systems=None, ports=None, services=None,
130
+ package_managers=None, frameworks=None, entrypoints=None,
131
+ commands=None, has_compose=0, has_devcontainer=0, has_multistage=0,
132
+ source_files=4, source_bytes=1024, document='container search doc',
133
+ document_hash='hash-1'):
134
+ return (
135
+ row, repo_id, repo_type, artifact_path, artifact_kind, author, sha,
136
+ created_at, last_modified, downloads, likes,
137
+ _json_text(tags or []), description, _json_text(artifact_files or []),
138
+ _json_text(qualification_reasons or []), _json_text(base_images or []),
139
+ _json_text(accelerators or []), _json_text(cuda_versions or []),
140
+ _json_text(rocm_versions or []), _json_text(python_versions or []),
141
+ _json_text(node_versions or []),
142
+ _json_text(operating_systems or []), _json_text(ports or []),
143
+ _json_text(services or []), _json_text(package_managers or []),
144
+ _json_text(frameworks or []), _json_text(entrypoints or []),
145
+ _json_text(commands or []), has_compose, has_devcontainer,
146
+ has_multistage, source_files, source_bytes, document, document_hash,
147
+ )
148
+
149
+
150
+ def basis(n, dim=4):
151
+ """n normalized basis vectors (row i -> dimension i % dim)."""
152
+ m = np.zeros((n, dim), dtype=np.float32)
153
+ for i in range(n):
154
+ m[i, i % dim] = 1.0
155
+ return m
156
+
157
+
158
+ def write_model_db(dir_path, rows):
159
+ conn = sqlite3.connect(Path(dir_path) / SQLITE_NAME)
160
+ conn.executescript(MODEL_DDL)
161
+ conn.executemany(
162
+ f"INSERT INTO models VALUES ({','.join('?' * 19)})", rows)
163
+ conn.commit()
164
+ conn.close()
165
+
166
+
167
+ def write_matrix(dir_path, vectors):
168
+ np.save(Path(dir_path) / MATRIX_NAME, vectors.astype(np.float16))
169
+
170
+
171
+ def write_kernel_db(dir_path, rows):
172
+ conn = sqlite3.connect(Path(dir_path) / SQLITE_NAME)
173
+ conn.executescript(KERNEL_DDL)
174
+ conn.executemany(
175
+ f"INSERT INTO kernels VALUES ({','.join('?' * 24)})", rows)
176
+ conn.commit()
177
+ conn.close()
178
+
179
+
180
+ def write_container_db(dir_path, rows):
181
+ conn = sqlite3.connect(Path(dir_path) / SQLITE_NAME)
182
+ conn.executescript(CONTAINER_DDL)
183
+ conn.executemany(
184
+ f"INSERT INTO containers VALUES ({','.join('?' * 35)})", rows)
185
+ conn.commit()
186
+ conn.close()
187
+
188
+
189
+ def write_index_dir(dir_path, rows, vectors):
190
+ write_model_db(dir_path, rows)
191
+ write_matrix(dir_path, vectors)
192
+
193
+
194
+ def make_spec(local_dir, **overrides):
195
+ base = dict(
196
+ key='models',
197
+ dataset_id='owner/corpus',
198
+ local_dir=Path(local_dir),
199
+ table='models',
200
+ id_column='model_id',
201
+ embed_model='text-embedding-3-small',
202
+ dimensions=4,
203
+ )
204
+ base.update(overrides)
205
+ return IndexSpec(**base)
206
+
207
+
208
+ FIXTURE_ROWS = [
209
+ model_row(0, 'indie/alpha', downloads=10, likes=1, tags='["llama","sft"]',
210
+ model_type='llama', license='mit', relation='finetune',
211
+ base_model='meta-llama/Llama-3-8B', month='2024-01',
212
+ created_at='2024-01-10T00:00:00Z', code_imports='["flash_attn"]',
213
+ architectures='["LlamaForCausalLM"]', params=8_000_000_000,
214
+ context_len=8192),
215
+ model_row(1, 'indie/beta', author='lab', downloads=500, likes=9,
216
+ tags='["mamba"]', model_type='mamba', license='apache-2.0',
217
+ month='2025-02', created_at='2025-02-02T00:00:00Z'),
218
+ model_row(2, 'indie/gamma', downloads=50, likes=3, tags='["qwen2","dpo"]',
219
+ model_type='qwen2', month='2024-03',
220
+ created_at='2024-03-05T00:00:00Z', num_experts=8,
221
+ code_imports='["triton"]', gated=1),
222
+ ]
223
+
224
+ EXCLUSION_ROWS = [
225
+ model_row(
226
+ 0, 'indie/llama-derivative', tags='["research"]',
227
+ base_model='meta-llama/Llama-3', model_type='transformer'),
228
+ model_row(
229
+ 1, 'indie/qwen-tagged', tags='["Qwen3"]',
230
+ model_type='transformer'),
231
+ model_row(
232
+ 2, 'indie/mistral-architecture',
233
+ architectures='["MistralForCausalLM"]', model_type='transformer'),
234
+ model_row(
235
+ 3, 'indie/gptq-repack', tags='["gptq"]',
236
+ config='{"quantization_config": {"bits": 4}}'),
237
+ model_row(
238
+ 4, 'indie/native-fp8-research', tags='["fp8"]',
239
+ model_type='research', config='{"torch_dtype": "float8_e4m3fn"}'),
240
+ model_row(
241
+ 5, 'indie/native-int8-research', tags='["int8"]',
242
+ model_type='research', config='{"torch_dtype": "int8"}'),
243
+ model_row(
244
+ 6, 'indie/config-null-quant', model_type='research',
245
+ config='{"quantization_config": null}'),
246
+ model_row(
247
+ 7, 'indie/malformed-config', model_type='research',
248
+ config='not-json'),
249
+ ]
250
+
251
+ KERNEL_ROWS = [
252
+ kernel_row(0, 'alice/fused-gemm'),
253
+ kernel_row(
254
+ 1, 'bob/paged-attention', author='bob', month='2024-06',
255
+ created_at='2024-06-02T00:00:00Z', downloads=20, likes=8,
256
+ languages='["triton","python"]', torch_versions='["2.4"]',
257
+ cuda_archs='["sm_80"]', dtypes='["fp32"]',
258
+ intrinsics='["tl.dot"]', kernel_names='["paged_attention"]',
259
+ torch_ops='["attention"]', build_variants='[]', variant_count=1,
260
+ has_build_dir=0, source_files=2, document='paged attention kernel'),
261
+ kernel_row(
262
+ 2, 'alice/int8-conv', month='2025-02',
263
+ created_at='2025-02-03T00:00:00Z', downloads=500, likes=5,
264
+ languages='["cpp"]', accelerators='["amd"]',
265
+ torch_versions='["2.5"]', cuda_archs='[]', cpu_archs='["x86_64"]',
266
+ dtypes='["int8"]', intrinsics='["avx2"]',
267
+ kernel_names='["int8_conv"]', torch_ops='["conv2d"]',
268
+ build_variants='["rocm"]', variant_count=5, has_build_dir=1,
269
+ source_files=6, document='int8 convolution kernel'),
270
+ ]
271
+
272
+ CONTAINER_ROWS = [
273
+ container_row(
274
+ 0, 'alice/cuda-recipe', repo_type='model', artifact_path='Dockerfile',
275
+ artifact_kind='recipe', author='alice', sha='sha-cuda',
276
+ created_at='2025-02-01T00:00:00Z',
277
+ last_modified='2025-02-02T00:00:00Z', downloads=300, likes=9,
278
+ tags=['docker', 'cuda'], description='CUDA PyTorch recipe',
279
+ artifact_files=['Dockerfile'],
280
+ qualification_reasons=['dockerfile', 'dependencies'],
281
+ base_images=['nvidia/cuda:12.8.0-runtime-ubuntu22.04'],
282
+ accelerators=['nvidia'], cuda_versions=['12.8'], rocm_versions=[],
283
+ python_versions=['3.11'], node_versions=[],
284
+ operating_systems=['ubuntu22.04'], ports=['7860'], services=['api'],
285
+ package_managers=['pip'], frameworks=['pytorch'],
286
+ entrypoints=['python app.py'], commands=['uvicorn app:app'],
287
+ has_compose=0, has_devcontainer=0, has_multistage=1, source_files=6,
288
+ source_bytes=4096,
289
+ document='python 3.11 cuda 12.8 pytorch recipe',
290
+ document_hash='hash-cuda'),
291
+ container_row(
292
+ 1, 'bob/rocm-recipe', repo_type='space',
293
+ artifact_path='docker/Dockerfile', artifact_kind='recipe',
294
+ author='bob', sha='sha-rocm', created_at='2025-03-01T00:00:00Z',
295
+ last_modified='2025-03-02T00:00:00Z', downloads=120, likes=4,
296
+ tags=['docker', 'rocm'], description='ROCm build',
297
+ artifact_files=['docker/Dockerfile'],
298
+ qualification_reasons=['dockerfile'],
299
+ base_images=['rocm/pytorch:latest'], accelerators=['amd'],
300
+ cuda_versions=[], rocm_versions=['6.1'], python_versions=['3.12'],
301
+ node_versions=['20'], operating_systems=['ubuntu24.04'],
302
+ ports=['7860'], services=[], package_managers=['apt', 'pip'],
303
+ frameworks=['pytorch'], entrypoints=[], commands=['python launch.py'],
304
+ has_compose=0, has_devcontainer=0, has_multistage=0, source_files=5,
305
+ source_bytes=3072, document='python 3.12 rocm pytorch recipe',
306
+ document_hash='hash-rocm'),
307
+ container_row(
308
+ 2, 'carol/compose-stack', repo_type='dataset',
309
+ artifact_path='compose/Dockerfile', artifact_kind='recipe',
310
+ author='carol', sha='sha-compose', created_at='2025-04-01T00:00:00Z',
311
+ last_modified='2025-04-02T00:00:00Z', downloads=80, likes=6,
312
+ tags=['docker', 'compose'], description='Compose-backed stack',
313
+ artifact_files=['compose/Dockerfile', 'compose/docker-compose.yml'],
314
+ qualification_reasons=['compose', 'devcontainer'],
315
+ base_images=['python:3.10-slim'], accelerators=[],
316
+ cuda_versions=[], rocm_versions=[], python_versions=['3.10'],
317
+ node_versions=['18'], operating_systems=['debian'],
318
+ ports=['5432', '8888'], services=['jupyter', 'postgres'],
319
+ package_managers=['apt', 'pip'], frameworks=['fastapi'],
320
+ entrypoints=['./boot.sh'], commands=['docker compose up'],
321
+ has_compose=1, has_devcontainer=1, has_multistage=0, source_files=8,
322
+ source_bytes=8192, document='compose jupyter postgres fastapi stack',
323
+ document_hash='hash-compose'),
324
+ container_row(
325
+ 3, 'dana/archive-bundle', repo_type='dataset',
326
+ artifact_path='artifacts/container.tar.gz', artifact_kind='archive',
327
+ author='dana', sha='sha-archive', created_at='2025-05-01T00:00:00Z',
328
+ last_modified='2025-05-02T00:00:00Z', downloads=40, likes=1,
329
+ tags=['docker', 'archive'], description='Archived CUDA image',
330
+ artifact_files=['artifacts/container.tar.gz'],
331
+ qualification_reasons=['archive'],
332
+ base_images=['ubuntu:22.04'], accelerators=['nvidia'],
333
+ cuda_versions=['12.8'], rocm_versions=[], python_versions=['3.11'],
334
+ node_versions=[], operating_systems=['ubuntu22.04'],
335
+ ports=['8080'], services=['worker'], package_managers=['pip'],
336
+ frameworks=['pytorch'], entrypoints=['./serve.sh'],
337
+ commands=['python worker.py'], has_compose=0, has_devcontainer=0,
338
+ has_multistage=0, source_files=2, source_bytes=2048,
339
+ document='python 3.11 cuda 12.8 archive image',
340
+ document_hash='hash-archive'),
341
+ ]
342
+
343
+
344
+ @pytest.fixture
345
+ def loaded_models(tmp_path):
346
+ write_index_dir(tmp_path, FIXTURE_ROWS, basis(3))
347
+ spec = make_spec(tmp_path)
348
+ index = LoadedIndex(spec, ModelAdapter(spec))
349
+ index.load()
350
+ assert index.ready.is_set() # fixture sanity
351
+ return index
352
+
353
+
354
+ @pytest.fixture
355
+ def loaded_exclusion_models(tmp_path):
356
+ write_index_dir(tmp_path, EXCLUSION_ROWS, basis(len(EXCLUSION_ROWS)))
357
+ spec = make_spec(tmp_path)
358
+ index = LoadedIndex(spec, ModelAdapter(spec))
359
+ index.load()
360
+ assert index.ready.is_set(), index.error
361
+ return index
362
+
363
+
364
+ def load_kernels(tmp_path):
365
+ from bsides.search_indexes import KernelAdapter
366
+
367
+ write_kernel_db(tmp_path, KERNEL_ROWS)
368
+ write_matrix(tmp_path, basis(3))
369
+ spec = make_spec(
370
+ tmp_path, key='kernels', dataset_id='owner/kernels', table='kernels',
371
+ id_column='repo_id')
372
+ index = LoadedIndex(spec, KernelAdapter(spec))
373
+ index.load()
374
+ assert index.ready.is_set(), index.error
375
+ return index
376
+
377
+
378
+ def load_containers(tmp_path):
379
+ from bsides.search_indexes import ContainerAdapter
380
+
381
+ write_container_db(tmp_path, CONTAINER_ROWS)
382
+ write_matrix(tmp_path, basis(len(CONTAINER_ROWS)))
383
+ spec = make_spec(
384
+ tmp_path, key='docker', dataset_id='owner/containers',
385
+ table='containers', id_column='repo_id')
386
+ index = LoadedIndex(spec, ContainerAdapter(spec))
387
+ index.load()
388
+ assert index.ready.is_set(), index.error
389
+ return index
390
+
391
+
392
+ @pytest.fixture
393
+ def loaded_containers(tmp_path):
394
+ return load_containers(tmp_path)
395
+
396
+
397
+ class FakeEmbeddings:
398
+ def __init__(self, vector):
399
+ self.vector = [float(v) for v in vector]
400
+ self.calls: list[dict] = []
401
+
402
+ def create(self, **kwargs):
403
+ self.calls.append(kwargs)
404
+ return SimpleNamespace(
405
+ data=[SimpleNamespace(embedding=list(self.vector))])
406
+
407
+
408
+ class FakeClient:
409
+ def __init__(self, vector):
410
+ self.embeddings = FakeEmbeddings(vector)
411
+
412
+
413
+ # ── download contract ─────────────────────────────────────────────────────────
414
+
415
+ def test_download_index_requests_dataset_files_into_spec_dir(tmp_path):
416
+ spec = make_spec(tmp_path / 'idx')
417
+ calls: list[tuple[str, dict]] = []
418
+
419
+ def downloader(dataset_id, **kwargs):
420
+ calls.append((dataset_id, kwargs))
421
+
422
+ download_index(downloader, spec)
423
+
424
+ assert calls == [
425
+ ('owner/corpus', {
426
+ 'repo_type': 'dataset',
427
+ 'local_dir': tmp_path / 'idx',
428
+ 'allow_patterns': ['embeddings.f16.npy', 'meta.sqlite'],
429
+ })
430
+ ]
431
+
432
+
433
+ # ── loader: happy paths ───────────────────────────────────────────────────────
434
+
435
+ def test_load_validates_marks_ready_and_computes_facets(loaded_models):
436
+ assert loaded_models.error is None
437
+ assert loaded_models.rows == 3
438
+ assert loaded_models.matrix.shape == (3, 4)
439
+ assert loaded_models.matrix.dtype == np.float32
440
+ assert loaded_models.facets['total'] == 3
441
+
442
+
443
+ def test_load_pulls_dataset_when_files_are_missing(tmp_path):
444
+ src = tmp_path / 'src'
445
+ src.mkdir()
446
+ write_index_dir(src, [model_row(0, 'a/b')], basis(1))
447
+ spec = make_spec(tmp_path / 'idx')
448
+ seen: list[tuple[str, dict]] = []
449
+
450
+ def downloader(dataset_id, **kwargs):
451
+ seen.append((dataset_id, dict(kwargs)))
452
+ target = Path(kwargs['local_dir'])
453
+ target.mkdir(parents=True, exist_ok=True)
454
+ for name in kwargs['allow_patterns']:
455
+ (target / name).write_bytes((src / name).read_bytes())
456
+
457
+ index = LoadedIndex(spec, ModelAdapter(spec))
458
+ index.load(downloader)
459
+
460
+ assert seen and seen[0][0] == 'owner/corpus'
461
+ assert index.ready.is_set() and index.error is None and index.rows == 1
462
+
463
+
464
+ def test_load_without_downloader_never_imports_hub_when_files_exist(tmp_path):
465
+ # Files already present β†’ the lazy huggingface_hub import never runs.
466
+ write_index_dir(tmp_path, [model_row(0, 'a/b')], basis(1))
467
+ index = LoadedIndex(make_spec(tmp_path))
468
+ index.load()
469
+ assert index.ready.is_set()
470
+
471
+
472
+
473
+ # ── loader: validation failures stay isolated ─────────────────────────────────
474
+
475
+ def test_load_rejects_matrix_rank_not_two(tmp_path):
476
+ write_model_db(tmp_path, [model_row(0, 'a/b')])
477
+ np.save(tmp_path / MATRIX_NAME, np.zeros(4, dtype=np.float16)) # 1-D
478
+ index = LoadedIndex(make_spec(tmp_path))
479
+ index.load()
480
+ assert not index.ready.is_set()
481
+ assert index.matrix is None
482
+ assert index.error and 'rank' in index.error
483
+
484
+
485
+ def test_load_rejects_dimension_mismatch(tmp_path):
486
+ write_model_db(tmp_path, [model_row(0, 'a/b')])
487
+ write_matrix(tmp_path, basis(1, dim=8)) # spec expects 4
488
+ index = LoadedIndex(make_spec(tmp_path))
489
+ index.load()
490
+ assert not index.ready.is_set()
491
+ assert index.error and 'dimensions' in index.error
492
+
493
+
494
+ def test_load_rejects_non_finite_values(tmp_path):
495
+ write_model_db(tmp_path, [model_row(0, 'a/b')])
496
+ m = basis(1)
497
+ m[0, 0] = np.inf
498
+ write_matrix(tmp_path, m)
499
+ index = LoadedIndex(make_spec(tmp_path))
500
+ index.load()
501
+ assert not index.ready.is_set()
502
+ assert index.error and 'non-finite' in index.error
503
+
504
+
505
+ def test_load_rejects_sqlite_that_fails_quick_check(tmp_path):
506
+ (tmp_path / SQLITE_NAME).write_bytes(b'this is not a sqlite database')
507
+ write_matrix(tmp_path, basis(1))
508
+ index = LoadedIndex(make_spec(tmp_path))
509
+ index.load()
510
+ assert not index.ready.is_set()
511
+ assert index.error
512
+
513
+
514
+ def test_load_rejects_row_count_mismatch(tmp_path):
515
+ write_model_db(tmp_path, [model_row(0, 'a/b')])
516
+ write_matrix(tmp_path, basis(2))
517
+ index = LoadedIndex(make_spec(tmp_path))
518
+ index.load()
519
+ assert not index.ready.is_set()
520
+ assert index.error and 'rows' in index.error
521
+
522
+
523
+ def test_load_rejects_rows_outside_matrix_bounds(tmp_path):
524
+ write_model_db(tmp_path,
525
+ [model_row(0, 'a/b'), model_row(5, 'c/d')]) # row 5 >= N=2
526
+ write_matrix(tmp_path, basis(2))
527
+ index = LoadedIndex(make_spec(tmp_path))
528
+ index.load()
529
+ assert not index.ready.is_set()
530
+ assert index.error and 'bounds' in index.error
531
+
532
+
533
+ def test_load_accepts_empty_index(tmp_path):
534
+ write_model_db(tmp_path, [])
535
+ write_matrix(tmp_path, np.zeros((0, 4), dtype=np.float32))
536
+ index = LoadedIndex(make_spec(tmp_path))
537
+ index.load()
538
+ assert index.ready.is_set() and index.rows == 0
539
+
540
+
541
+ def test_load_isolates_downloader_failure(tmp_path):
542
+ def downloader(dataset_id, **kwargs):
543
+ raise RuntimeError('hub says no')
544
+
545
+ index = LoadedIndex(make_spec(tmp_path))
546
+ index.load(downloader)
547
+ assert not index.ready.is_set()
548
+ assert index.error and 'hub says no' in index.error
549
+
550
+
551
+
552
+ # ── reload: hot-swap contract ─────────────────────────────────────────────────
553
+
554
+ def republish(dir_path, rows, vectors):
555
+ """A downloader that replaces the corpus files, like a fresh publish."""
556
+ def downloader(dataset_id, **kwargs):
557
+ target = Path(kwargs['local_dir'])
558
+ for name in (SQLITE_NAME, MATRIX_NAME):
559
+ (target / name).unlink(missing_ok=True)
560
+ write_index_dir(target, rows, vectors)
561
+ return downloader
562
+
563
+
564
+ def test_reload_swaps_in_the_freshly_published_index(loaded_models):
565
+ assert loaded_models.rows == 3
566
+
567
+ swapped = loaded_models.reload(republish(
568
+ loaded_models.spec.local_dir,
569
+ [model_row(0, 'indie/delta', tags='["rwkv"]', model_type='rwkv'),
570
+ model_row(1, 'indie/epsilon', tags='["mamba"]', model_type='mamba')],
571
+ basis(2)))
572
+
573
+ assert swapped is True
574
+ assert loaded_models.error is None
575
+ assert loaded_models.ready.is_set()
576
+ assert loaded_models.rows == 2
577
+ assert loaded_models.matrix.shape == (2, 4)
578
+ ids = [r[0] for r in loaded_models.db().execute(
579
+ 'SELECT model_id FROM models ORDER BY row')]
580
+ assert ids == ['indie/delta', 'indie/epsilon']
581
+ assert loaded_models.facets['total'] == 2
582
+ assert {a['v'] for a in loaded_models.facets['archs']} == {'rwkv', 'mamba'}
583
+
584
+
585
+ def test_reload_invalidates_connections_to_the_replaced_file(loaded_models):
586
+ stale = loaded_models.db()
587
+ generation = loaded_models.generation
588
+
589
+ loaded_models.reload(republish(
590
+ loaded_models.spec.local_dir,
591
+ [model_row(0, 'indie/delta')], basis(1)))
592
+
593
+ assert loaded_models.generation > generation
594
+ fresh = loaded_models.db()
595
+ assert fresh is not stale
596
+ assert fresh.execute(
597
+ 'SELECT model_id FROM models').fetchone()[0] == 'indie/delta'
598
+
599
+
600
+ def test_reload_does_not_skip_the_download_step(loaded_models):
601
+ # load() skips the downloader call when files exist; reload() must not, or
602
+ # a newly published revision would never be checked for changed files.
603
+ seen: list[str] = []
604
+
605
+ def downloader(dataset_id, **kwargs):
606
+ seen.append(dataset_id)
607
+
608
+ assert loaded_models.reload(downloader) is True
609
+ assert seen == ['owner/corpus']
610
+
611
+
612
+ def test_failed_download_keeps_serving_the_previous_index(loaded_models):
613
+ def downloader(dataset_id, **kwargs):
614
+ raise RuntimeError('hub says no')
615
+
616
+ assert loaded_models.reload(downloader) is False
617
+ assert loaded_models.ready.is_set() # never went down
618
+ assert loaded_models.rows == 3
619
+ assert loaded_models.matrix.shape == (3, 4)
620
+ assert loaded_models.facets['total'] == 3
621
+ assert 'hub says no' in loaded_models.error
622
+ assert loaded_models.db().execute(
623
+ 'SELECT COUNT(*) FROM models').fetchone()[0] == 3
624
+
625
+
626
+ def test_reload_rejecting_a_bad_publish_keeps_serving_the_old_index(
627
+ loaded_models):
628
+ # A published matrix at the wrong width must not replace a good one.
629
+ assert loaded_models.reload(republish(
630
+ loaded_models.spec.local_dir,
631
+ [model_row(0, 'indie/delta')], basis(1, dim=8))) is False
632
+
633
+ assert loaded_models.ready.is_set()
634
+ assert loaded_models.rows == 3
635
+ assert loaded_models.matrix.shape == (3, 4)
636
+ assert 'dimensions' in loaded_models.error
637
+
638
+
639
+ def test_reload_rejects_a_publish_whose_rows_do_not_match_the_matrix(
640
+ loaded_models):
641
+ assert loaded_models.reload(republish(
642
+ loaded_models.spec.local_dir,
643
+ [model_row(0, 'indie/delta')], basis(2))) is False
644
+
645
+ assert loaded_models.ready.is_set() and loaded_models.rows == 3
646
+ assert 'rows' in loaded_models.error
647
+
648
+
649
+ # ── registry ──────────────────────────────────────────────────────────────────
650
+
651
+ def test_registry_isolates_a_broken_corpus_from_a_healthy_one(tmp_path):
652
+ good_dir, bad_dir = tmp_path / 'good', tmp_path / 'bad'
653
+ good_dir.mkdir()
654
+ bad_dir.mkdir()
655
+ write_index_dir(good_dir, FIXTURE_ROWS, basis(3))
656
+ write_model_db(bad_dir, [model_row(0, 'a/b')])
657
+ write_matrix(bad_dir, basis(1, dim=8)) # wrong dims β†’ validation failure
658
+
659
+ registry = IndexRegistry()
660
+ registry.register(ModelAdapter(make_spec(good_dir, key='models')))
661
+ registry.register(ModelAdapter(make_spec(bad_dir, key='kernels')))
662
+ registry.load_all() # must not raise
663
+
664
+ assert registry.index('models').ready.is_set()
665
+ bad = registry.index('kernels')
666
+ assert not bad.ready.is_set() and bad.error
667
+
668
+ status = registry.status()
669
+ assert status['models']['ready'] is True
670
+ assert status['models']['rows'] == 3
671
+ assert status['models']['error'] is None
672
+ assert status['kernels']['ready'] is False
673
+ assert status['kernels']['error']
674
+
675
+
676
+ def test_registry_unknown_key_raises_keyerror():
677
+ with pytest.raises(KeyError):
678
+ IndexRegistry().index('bogus')
679
+
680
+
681
+ def test_registry_start_loading_runs_isolated_threads(tmp_path):
682
+ good_dir = tmp_path / 'good'
683
+ good_dir.mkdir()
684
+ write_index_dir(good_dir, FIXTURE_ROWS, basis(3))
685
+ registry = IndexRegistry()
686
+ registry.register(ModelAdapter(make_spec(good_dir, key='models')))
687
+ registry.start_loading()
688
+ index = registry.index('models')
689
+ assert index.ready.wait(timeout=10)
690
+ assert index.error is None
691
+
692
+
693
+ # ── LoadedIndex connections ───────────────────────────────────────────────────
694
+
695
+ def test_db_is_thread_local_readonly_and_generation_scoped(loaded_models):
696
+ conn = loaded_models.db()
697
+ assert conn is loaded_models.db() # same thread β†’ same connection
698
+ with pytest.raises(sqlite3.OperationalError):
699
+ conn.execute('CREATE TABLE hack (x INTEGER)')
700
+
701
+ loaded_models.invalidate_connections()
702
+ fresh = loaded_models.db()
703
+ assert fresh is not conn
704
+ assert fresh.execute('SELECT COUNT(*) FROM models').fetchone()[0] == 3
705
+
706
+
707
+ # ── query embedding ───────────────────────────────────────────────────────────
708
+
709
+ def test_embed_query_uses_spec_model_and_selected_matrix_dimensions(
710
+ loaded_models):
711
+ client = FakeClient([2.0, 0.0, 0.0, 0.0])
712
+ vector = embed_query(loaded_models, client, 'odd attention')
713
+
714
+ assert vector.shape == (4,)
715
+ assert np.isclose(np.linalg.norm(vector), 1.0)
716
+ assert client.embeddings.calls == [{
717
+ 'model': 'text-embedding-3-small',
718
+ 'dimensions': 4,
719
+ 'input': 'odd attention',
720
+ }]
721
+
722
+
723
+ def test_embed_query_caps_input_at_max_chars(loaded_models):
724
+ client = FakeClient([1, 0, 0, 0])
725
+ embed_query(loaded_models, client, 'x' * (MAX_QUERY_CHARS + 250))
726
+ assert len(client.embeddings.calls[0]['input']) == MAX_QUERY_CHARS
727
+
728
+
729
+ def test_embed_query_without_client_names_the_selected_credential(loaded_models):
730
+ with pytest.raises(
731
+ UnavailableError, match='no OPENAI_KERNEL_API_KEY'):
732
+ embed_query(
733
+ loaded_models, None, 'q',
734
+ credential_name='OPENAI_KERNEL_API_KEY')
735
+
736
+
737
+ def test_embed_query_without_matrix_is_unavailable(tmp_path):
738
+ index = LoadedIndex(make_spec(tmp_path))
739
+ with pytest.raises(UnavailableError, match='index not loaded'):
740
+ embed_query(index, FakeClient([1, 0, 0, 0]), 'q')
741
+
742
+
743
+
744
+ # ── shared semantic ranking ───────────────────────────────────────────────────
745
+
746
+ def test_semantic_rank_scores_orders_and_paginates(loaded_models):
747
+ client = FakeClient([0.2, 1.0, 0.5, 0.0]) # beta > gamma > alpha
748
+ total, cards = semantic_rank(
749
+ loaded_models.db(), loaded_models, loaded_models.adapter,
750
+ where='1=1', args=[], sem_q='mamba hybrid', page=0, page_size=2,
751
+ embed_client=client)
752
+
753
+ assert total == 3
754
+ assert [c['model_id'] for c in cards] == ['indie/beta', 'indie/gamma']
755
+ assert cards[0]['score'] > cards[1]['score']
756
+ assert cards[0]['tags'] == ['mamba'] # JSON columns decoded
757
+
758
+
759
+ def test_semantic_rank_uses_candidate_scoring_for_small_filtered_sets(
760
+ loaded_models):
761
+ # total=1 < N//2 β†’ MAT[rows] @ qv branch instead of (MAT @ qv)[rows]
762
+ client = FakeClient([0.0, 0.0, 1.0, 0.0])
763
+ total, cards = semantic_rank(
764
+ loaded_models.db(), loaded_models, loaded_models.adapter,
765
+ where='model_id = ?', args=['indie/gamma'], sem_q='q',
766
+ page=0, page_size=24, embed_client=client)
767
+
768
+ assert total == 1
769
+ assert [c['model_id'] for c in cards] == ['indie/gamma']
770
+ assert cards[0]['score'] == 1.0
771
+
772
+
773
+ def test_semantic_rank_empty_candidates_skip_embedding(loaded_models):
774
+ client = FakeClient([1, 0, 0, 0])
775
+ total, cards = semantic_rank(
776
+ loaded_models.db(), loaded_models, loaded_models.adapter,
777
+ where='row < 0', args=[], sem_q='q', page=0, page_size=24,
778
+ embed_client=client)
779
+
780
+ assert (total, cards) == (0, [])
781
+ assert client.embeddings.calls == []
782
+
783
+
784
+ def test_semantic_rank_page_beyond_window_returns_total_but_no_cards(
785
+ loaded_models):
786
+ client = FakeClient([1, 0, 0, 0])
787
+ total, cards = semantic_rank(
788
+ loaded_models.db(), loaded_models, loaded_models.adapter,
789
+ where='1=1', args=[], sem_q='q', page=3, page_size=2,
790
+ embed_client=client)
791
+
792
+ assert total == 3 and cards == []
793
+
794
+
795
+ def test_semantic_rank_propagates_unavailable_embedding(loaded_models):
796
+ with pytest.raises(UnavailableError):
797
+ semantic_rank(
798
+ loaded_models.db(), loaded_models, loaded_models.adapter,
799
+ where='1=1', args=[], sem_q='q', page=0, page_size=2,
800
+ embed_client=None)
801
+
802
+
803
+ # ── shared structured paging ──────────────────────────────────────────────────
804
+
805
+ def test_structured_page_orders_counts_and_paginates(loaded_models):
806
+ total, cards = structured_page(
807
+ loaded_models.db(), loaded_models, loaded_models.adapter,
808
+ where='1=1', args=[], sort='downloads', page=0, page_size=2)
809
+
810
+ assert total == 3
811
+ assert [c['model_id'] for c in cards] == ['indie/beta', 'indie/gamma']
812
+
813
+ total2, cards2 = structured_page(
814
+ loaded_models.db(), loaded_models, loaded_models.adapter,
815
+ where='1=1', args=[], sort='downloads', page=1, page_size=2)
816
+ assert total2 == 3
817
+ assert [c['model_id'] for c in cards2] == ['indie/alpha']
818
+
819
+
820
+ def test_structured_page_unknown_sort_falls_back_to_newest(loaded_models):
821
+ total, cards = structured_page(
822
+ loaded_models.db(), loaded_models, loaded_models.adapter,
823
+ where='1=1', args=[], sort='relevance', page=0, page_size=24)
824
+
825
+ assert total == 3
826
+ assert [c['model_id'] for c in cards] == [
827
+ 'indie/beta', 'indie/gamma', 'indie/alpha'] # created_at DESC
828
+
829
+
830
+
831
+ # ── ModelAdapter: request parsing ─────────────────────────────────────────────
832
+
833
+ def test_extract_constraints_pulls_dates_params_downloads_and_flags(
834
+ loaded_models):
835
+ r = SearchReq(query='mamba hybrid feb 2025 under 3b params '
836
+ '10-50 downloads dpo custom code')
837
+ sem_q = loaded_models.adapter.extract_constraints(r)
838
+
839
+ assert r.month_from == r.month_to == '2025-02'
840
+ assert r.params_max == 3_000_000_000
841
+ assert r.downloads_min == 10 and r.downloads_max == 50
842
+ assert r.custom_code_only is True
843
+ assert 'dpo' in r.methods
844
+ # constraints leave the embedding text; meaning stays
845
+ assert 'feb' not in sem_q and '2025' not in sem_q
846
+ assert 'mamba' in sem_q and 'hybrid' in sem_q and 'dpo' in sem_q
847
+
848
+
849
+ def test_extract_constraints_never_loosens_explicit_filters(loaded_models):
850
+ r = SearchReq(query='2024 mamba', month_from='2023-06', methods=['sft'])
851
+ loaded_models.adapter.extract_constraints(r)
852
+
853
+ assert r.month_from == '2023-06' # explicit value untouched
854
+ assert r.month_to == '2024-12' # empty slot filled
855
+ assert r.methods == ['sft']
856
+
857
+
858
+ def test_extract_constraints_empty_remainder_skips_embedding(loaded_models):
859
+ r = SearchReq(query='under 3b params')
860
+ assert loaded_models.adapter.extract_constraints(r) == ''
861
+ assert r.params_max == 3_000_000_000
862
+
863
+
864
+ # ── ModelAdapter: filtering, serialization, facets ────────────────────────────
865
+
866
+ def test_model_adapter_exposes_spec_result_key_and_card_columns(loaded_models):
867
+ adapter = loaded_models.adapter
868
+ assert adapter.result_key == 'models'
869
+ assert adapter.card_columns == (
870
+ 'row, model_id, author, month, downloads, likes, tags, base_model, '
871
+ 'relation, license, params, model_type, architectures, context_len, '
872
+ 'num_experts, code_imports, gated')
873
+ assert adapter.sort_order('downloads') == 'downloads DESC'
874
+ assert adapter.sort_order('bogus') == 'created_at DESC'
875
+
876
+
877
+ def test_build_where_empty_request_is_neutral(loaded_models):
878
+ where, args = loaded_models.adapter.build_where(SearchReq())
879
+ assert where == '1=1' and args == []
880
+
881
+
882
+ def test_model_exclusions_match_family_evidence(loaded_exclusion_models):
883
+ request = SearchReq(exclude_families=['llama', 'qwen'])
884
+ where, args = loaded_exclusion_models.adapter.build_where(request)
885
+ ids = [r[0] for r in loaded_exclusion_models.db().execute(
886
+ f'SELECT model_id FROM models WHERE {where}', args)]
887
+
888
+ assert 'indie/llama-derivative' not in ids
889
+ assert 'indie/qwen-tagged' not in ids
890
+ assert 'indie/mistral-architecture' in ids
891
+
892
+
893
+ def test_model_family_exclusion_matches_expanded_mistral_terms(
894
+ loaded_exclusion_models):
895
+ where, args = loaded_exclusion_models.adapter.build_where(
896
+ SearchReq(exclude_families=['mistral']))
897
+ ids = [r[0] for r in loaded_exclusion_models.db().execute(
898
+ f'SELECT model_id FROM models WHERE {where}', args)]
899
+
900
+ assert 'indie/mistral-architecture' not in ids
901
+
902
+
903
+ def test_quantization_exclusion_requires_strong_evidence(
904
+ loaded_exclusion_models):
905
+ request = SearchReq(exclude_quantizations=True)
906
+ where, args = loaded_exclusion_models.adapter.build_where(request)
907
+ ids = [r[0] for r in loaded_exclusion_models.db().execute(
908
+ f'SELECT model_id FROM models WHERE {where}', args)]
909
+
910
+ assert 'indie/gptq-repack' not in ids
911
+ assert 'indie/native-fp8-research' in ids
912
+ assert 'indie/native-int8-research' in ids
913
+ assert 'indie/config-null-quant' in ids
914
+ assert 'indie/malformed-config' in ids
915
+
916
+
917
+ def test_malformed_config_remains_searchable_during_quantization_exclusion(
918
+ loaded_exclusion_models):
919
+ request = SearchReq(exclude_quantizations=True)
920
+ where, args = loaded_exclusion_models.adapter.build_where(request)
921
+ assert (
922
+ "CASE WHEN json_valid(config) "
923
+ "THEN json_extract(config, '$.quantization_config') IS NOT NULL "
924
+ "ELSE 0 END" in where)
925
+ ids = [r[0] for r in loaded_exclusion_models.db().execute(
926
+ f'SELECT model_id FROM models WHERE {where}', args)]
927
+
928
+ assert 'indie/malformed-config' in ids
929
+
930
+
931
+ def test_model_exclusions_are_neutral_by_default(loaded_models):
932
+ request = SearchReq()
933
+
934
+ assert request.exclude_families == []
935
+ assert request.exclude_quantizations is False
936
+ where, args = loaded_models.adapter.build_where(request)
937
+ assert where == '1=1'
938
+ assert args == []
939
+
940
+
941
+ def test_build_where_covers_every_model_filter(loaded_models):
942
+ r = SearchReq(
943
+ month_from='2024-01', month_to='2024-12', archs=['mamba', 'llama'],
944
+ methods=['sft'], relation='finetune', base_model='llama',
945
+ author='indie', license='mit', imports=['flash_attn'],
946
+ params_min=1, params_max=2, context_min=4096, downloads_min=5,
947
+ downloads_max=50, moe_only=True, custom_code_only=True,
948
+ config_contains='rope')
949
+ where, args = loaded_models.adapter.build_where(r)
950
+
951
+ assert where == (
952
+ '1=1 AND month >= ? AND month <= ? '
953
+ 'AND ((model_type = ? OR tags LIKE ?) OR (model_type = ? OR tags LIKE ?)) '
954
+ 'AND tags LIKE ? AND relation = ? AND base_model LIKE ? AND author LIKE ? '
955
+ 'AND license = ? AND code_imports LIKE ? AND params >= ? AND params <= ? '
956
+ 'AND context_len >= ? AND downloads >= ? AND downloads <= ? '
957
+ 'AND num_experts > 1 '
958
+ "AND (code_imports IS NOT NULL OR tags LIKE '%\"custom_code\"%') "
959
+ 'AND config LIKE ?')
960
+ assert args == ['2024-01', '2024-12', 'mamba', '%"mamba"%', 'llama',
961
+ '%"llama"%', '%"sft"%', 'finetune', '%llama%', '%indie%',
962
+ 'mit', '%"flash_attn"%', 1, 2, 4096, 5, 50, '%rope%']
963
+
964
+
965
+ def test_build_where_executes_against_real_sqlite(loaded_models):
966
+ where, args = loaded_models.adapter.build_where(SearchReq(archs=['mamba']))
967
+ n = loaded_models.db().execute(
968
+ f'SELECT COUNT(*) FROM models WHERE {where}', args).fetchone()[0]
969
+ assert n == 1
970
+
971
+
972
+ def test_card_decodes_json_columns_and_rounds_score(loaded_models):
973
+ row = loaded_models.db().execute(
974
+ f'SELECT {loaded_models.adapter.card_columns} FROM models '
975
+ 'WHERE row = 0').fetchone()
976
+ d = loaded_models.adapter.card(row, 0.87654)
977
+
978
+ assert d['tags'] == ['llama', 'sft']
979
+ assert d['code_imports'] == ['flash_attn']
980
+ assert d['architectures'] == ['LlamaForCausalLM']
981
+ assert d['score'] == 0.8765
982
+
983
+
984
+ def test_card_without_score_omits_it(loaded_models):
985
+ row = loaded_models.db().execute(
986
+ f'SELECT {loaded_models.adapter.card_columns} FROM models '
987
+ 'WHERE row = 1').fetchone()
988
+ assert 'score' not in loaded_models.adapter.card(row)
989
+
990
+
991
+ def test_compute_facets_shape_matches_the_legacy_payload(loaded_models):
992
+ f = loaded_models.facets
993
+
994
+ assert f['total'] == 3
995
+ assert f['months'] == ['2024-01', '2024-03', '2025-02']
996
+ assert f['enriched'] is True
997
+ assert {a['v']: a['n'] for a in f['archs']} == {
998
+ 'llama': 1, 'mamba': 1, 'qwen2': 1}
999
+ assert {m['v']: m['n'] for m in f['methods']} == {'sft': 1, 'dpo': 1}
1000
+ assert {l['v'] for l in f['licenses']} == {'mit', 'apache-2.0'}
1001
+ assert {r['v'] for r in f['relations']} == {'finetune'}
1002
+ assert {i['v'] for i in f['imports']} == {'flash_attn', 'triton'}
1003
+ assert {a['v'] for a in f['authors']} == {'indie', 'lab'}
1004
+
1005
+
1006
+ # ── KernelAdapter: isolated request, filtering, cards and facets ──────────────
1007
+
1008
+ def test_search_request_keeps_optional_adapter_filters():
1009
+ request = SearchReq(filters={'languages': ['cuda']})
1010
+
1011
+ assert request.filters == {'languages': ['cuda']}
1012
+
1013
+
1014
+ def test_model_adapter_ignores_adapter_filters(loaded_models):
1015
+ request = SearchReq(filters={
1016
+ 'languages': ['cuda'], 'has_build_dir': True,
1017
+ 'variant_count_min': 99,
1018
+ })
1019
+
1020
+ where, args = loaded_models.adapter.build_where(request)
1021
+
1022
+ assert where == '1=1'
1023
+ assert args == []
1024
+
1025
+
1026
+ def test_kernel_filters_execute_against_real_sqlite(tmp_path):
1027
+ index = load_kernels(tmp_path)
1028
+ request = SearchReq(filters={
1029
+ 'author': 'ali',
1030
+ 'month_from': '2025-01',
1031
+ 'month_to': '2025-01',
1032
+ 'downloads_min': 50,
1033
+ 'downloads_max': 200,
1034
+ 'languages': ['cuda', 'triton'],
1035
+ 'accelerators': ['nvidia'],
1036
+ 'cuda_archs': ['sm_90'],
1037
+ 'dtypes': ['bf16'],
1038
+ 'intrinsics': ['mma_sync'],
1039
+ 'torch_ops': ['matmul'],
1040
+ 'torch_versions': ['2.5'],
1041
+ 'has_build_dir': True,
1042
+ 'variant_count_min': 3,
1043
+ 'not_a_kernel_filter': 'ignored',
1044
+ })
1045
+
1046
+ where, args = index.adapter.build_where(request)
1047
+ ids = [row[0] for row in index.db().execute(
1048
+ f'SELECT repo_id FROM kernels WHERE {where}', args)]
1049
+
1050
+ assert ids == ['alice/fused-gemm']
1051
+
1052
+
1053
+ def test_kernel_array_filter_values_are_or_within_one_category(tmp_path):
1054
+ index = load_kernels(tmp_path)
1055
+ where, args = index.adapter.build_where(SearchReq(filters={
1056
+ 'languages': ['cuda', 'triton'],
1057
+ }))
1058
+
1059
+ ids = [row[0] for row in index.db().execute(
1060
+ f'SELECT repo_id FROM kernels WHERE {where} ORDER BY row', args)]
1061
+
1062
+ assert ids == ['alice/fused-gemm', 'bob/paged-attention']
1063
+
1064
+
1065
+ def test_kernel_adapter_ignores_legacy_model_and_unknown_filters(tmp_path):
1066
+ index = load_kernels(tmp_path)
1067
+ request = SearchReq(
1068
+ author='nobody', downloads_min=9999, archs=['mamba'],
1069
+ filters={'license': 'mit', 'cpu_archs': ['x86_64']})
1070
+
1071
+ where, args = index.adapter.build_where(request)
1072
+ count = index.db().execute(
1073
+ f'SELECT COUNT(*) FROM kernels WHERE {where}', args).fetchone()[0]
1074
+
1075
+ assert count == 3
1076
+
1077
+
1078
+ def test_kernel_card_decodes_arrays_and_returns_facts(tmp_path):
1079
+ index = load_kernels(tmp_path)
1080
+ row = index.db().execute(
1081
+ f'SELECT {index.adapter.card_columns} FROM kernels WHERE row = 0'
1082
+ ).fetchone()
1083
+
1084
+ card = index.adapter.card(row, 0.87654)
1085
+
1086
+ assert card['repo_id'] == 'alice/fused-gemm'
1087
+ assert card['tags'] == ['kernels']
1088
+ assert card['languages'] == ['cuda', 'python']
1089
+ assert card['accelerators'] == ['nvidia']
1090
+ assert card['torch_versions'] == ['2.5']
1091
+ assert card['cuda_archs'] == ['sm_90']
1092
+ assert card['cpu_archs'] == []
1093
+ assert card['operating_systems'] == ['linux']
1094
+ assert card['dtypes'] == ['bf16', 'fp16']
1095
+ assert card['intrinsics'] == ['mma_sync', '__syncthreads']
1096
+ assert card['kernel_names'] == ['fused_gemm']
1097
+ assert card['torch_ops'] == ['matmul']
1098
+ assert card['build_variants'] == ['torch25-cu124']
1099
+ assert card['variant_count'] == 3
1100
+ assert card['has_build_dir'] is True
1101
+ assert card['source_files'] == 4
1102
+ assert card['score'] == 0.8765
1103
+ assert 'document' not in card
1104
+
1105
+
1106
+ def test_kernel_facets_count_members_of_json_arrays(tmp_path):
1107
+ facets = load_kernels(tmp_path).facets
1108
+
1109
+ assert facets['total'] == 3
1110
+ assert facets['months'] == ['2024-06', '2025-01', '2025-02']
1111
+ assert {x['v']: x['n'] for x in facets['authors']} == {
1112
+ 'alice': 2, 'bob': 1}
1113
+ assert {x['v']: x['n'] for x in facets['languages']} == {
1114
+ 'python': 2, 'cpp': 1, 'cuda': 1, 'triton': 1}
1115
+ assert {x['v']: x['n'] for x in facets['accelerators']} == {
1116
+ 'nvidia': 2, 'amd': 1}
1117
+ assert {x['v']: x['n'] for x in facets['torch_versions']} == {
1118
+ '2.5': 2, '2.4': 1}
1119
+ assert {x['v']: x['n'] for x in facets['cuda_archs']} == {
1120
+ 'sm_80': 1, 'sm_90': 1}
1121
+ assert {x['v']: x['n'] for x in facets['dtypes']} == {
1122
+ 'bf16': 1, 'fp16': 1, 'fp32': 1, 'int8': 1}
1123
+ assert {x['v']: x['n'] for x in facets['intrinsics']} == {
1124
+ '__syncthreads': 1, 'avx2': 1, 'mma_sync': 1, 'tl.dot': 1}
1125
+ assert {x['v']: x['n'] for x in facets['torch_ops']} == {
1126
+ 'attention': 1, 'conv2d': 1, 'matmul': 1}
1127
+ assert facets['has_build_dir'] == [
1128
+ {'v': True, 'n': 2}, {'v': False, 'n': 1}]
1129
+
1130
+
1131
+ def test_kernel_structured_page_uses_kernel_envelope_contract(tmp_path):
1132
+ index = load_kernels(tmp_path)
1133
+ where, args = index.adapter.build_where(SearchReq(filters={
1134
+ 'accelerators': ['nvidia'],
1135
+ }))
1136
+
1137
+ total, cards = structured_page(
1138
+ index.db(), index, index.adapter, where=where, args=args,
1139
+ sort='downloads', page=0, page_size=24)
1140
+
1141
+ assert index.adapter.result_key == 'kernels'
1142
+ assert total == 2
1143
+ assert [card['repo_id'] for card in cards] == [
1144
+ 'alice/fused-gemm', 'bob/paged-attention']
1145
+
1146
+
1147
+ def test_kernel_query_text_is_preserved_for_semantic_search(tmp_path):
1148
+ index = load_kernels(tmp_path)
1149
+
1150
+ assert index.adapter.extract_constraints(
1151
+ SearchReq(query=' fused attention ')) == 'fused attention'
1152
+
1153
+
1154
+ # ── ContainerAdapter: request, filtering, cards and facets ───────────────────
1155
+
1156
+ def test_container_defaults_to_requested_recipe_filter(loaded_containers):
1157
+ request = SearchReq(filters={'artifact_kinds': ['recipe']})
1158
+ where, args = loaded_containers.adapter.build_where(request)
1159
+ kinds = {
1160
+ row[0] for row in loaded_containers.db().execute(
1161
+ f'SELECT artifact_kind FROM containers WHERE {where}', args)
1162
+ }
1163
+
1164
+ assert kinds == {'recipe'}
1165
+
1166
+
1167
+ def test_container_filters_are_anded_across_groups_and_ored_within_group(
1168
+ loaded_containers):
1169
+ request = SearchReq(filters={
1170
+ 'artifact_kinds': ['recipe'],
1171
+ 'python_versions': ['3.11', '3.12'],
1172
+ 'accelerators': ['nvidia', 'amd'],
1173
+ 'cuda_versions': ['12.8'],
1174
+ 'frameworks': ['pytorch'],
1175
+ 'has_multistage': True,
1176
+ })
1177
+ where, args = loaded_containers.adapter.build_where(request)
1178
+ ids = [row[0] for row in loaded_containers.db().execute(
1179
+ f'SELECT repo_id FROM containers WHERE {where}', args)]
1180
+
1181
+ assert ids == ['alice/cuda-recipe']
1182
+
1183
+
1184
+ def test_container_archive_rows_appear_when_artifact_kind_filter_is_absent(
1185
+ loaded_containers):
1186
+ request = SearchReq(filters={
1187
+ 'python_versions': ['3.11'],
1188
+ 'cuda_versions': ['12.8'],
1189
+ 'frameworks': ['pytorch'],
1190
+ })
1191
+ where, args = loaded_containers.adapter.build_where(request)
1192
+ ids = [row[0] for row in loaded_containers.db().execute(
1193
+ f'SELECT repo_id FROM containers WHERE {where} ORDER BY row', args)]
1194
+
1195
+ assert ids == ['alice/cuda-recipe', 'dana/archive-bundle']
1196
+
1197
+
1198
+ def test_container_scalar_boolean_and_json_filters_execute_against_sqlite(
1199
+ loaded_containers):
1200
+ request = SearchReq(filters={
1201
+ 'repo_types': ['dataset'],
1202
+ 'services': ['jupyter', 'postgres'],
1203
+ 'package_managers': ['apt'],
1204
+ 'ports': ['8888'],
1205
+ 'has_compose': True,
1206
+ 'has_devcontainer': True,
1207
+ })
1208
+ where, args = loaded_containers.adapter.build_where(request)
1209
+ ids = [row[0] for row in loaded_containers.db().execute(
1210
+ f'SELECT repo_id FROM containers WHERE {where}', args)]
1211
+
1212
+ assert ids == ['carol/compose-stack']
1213
+
1214
+
1215
+ def test_container_query_text_is_preserved_for_semantic_search(
1216
+ loaded_containers):
1217
+ assert loaded_containers.adapter.extract_constraints(
1218
+ SearchReq(query=' python 3.11 cuda 12.8 ')
1219
+ ) == 'python 3.11 cuda 12.8'
1220
+
1221
+
1222
+ def test_container_card_decodes_json_arrays_and_preserves_empty_arrays(
1223
+ loaded_containers):
1224
+ row = loaded_containers.db().execute(
1225
+ f'SELECT {loaded_containers.adapter.card_columns} FROM containers '
1226
+ 'WHERE row = 1').fetchone()
1227
+
1228
+ card = loaded_containers.adapter.card(row, 0.87654)
1229
+
1230
+ assert card['repo_id'] == 'bob/rocm-recipe'
1231
+ assert card['repo_type'] == 'space'
1232
+ assert card['artifact_path'] == 'docker/Dockerfile'
1233
+ assert card['artifact_kind'] == 'recipe'
1234
+ assert card['tags'] == ['docker', 'rocm']
1235
+ assert card['artifact_files'] == ['docker/Dockerfile']
1236
+ assert card['qualification_reasons'] == ['dockerfile']
1237
+ assert card['base_images'] == ['rocm/pytorch:latest']
1238
+ assert card['accelerators'] == ['amd']
1239
+ assert card['cuda_versions'] == []
1240
+ assert card['rocm_versions'] == ['6.1']
1241
+ assert card['python_versions'] == ['3.12']
1242
+ assert card['node_versions'] == ['20']
1243
+ assert card['operating_systems'] == ['ubuntu24.04']
1244
+ assert card['ports'] == ['7860']
1245
+ assert card['services'] == []
1246
+ assert card['package_managers'] == ['apt', 'pip']
1247
+ assert card['frameworks'] == ['pytorch']
1248
+ assert card['entrypoints'] == []
1249
+ assert card['commands'] == ['python launch.py']
1250
+ assert card['has_compose'] is False
1251
+ assert card['has_devcontainer'] is False
1252
+ assert card['has_multistage'] is False
1253
+ assert card['score'] == 0.8765
1254
+ assert 'document' not in card
1255
+
1256
+
1257
+ def test_container_facets_count_scalars_arrays_and_booleans(
1258
+ loaded_containers):
1259
+ facets = loaded_containers.facets
1260
+
1261
+ assert facets['total'] == 4
1262
+ assert {item['v']: item['n'] for item in facets['repo_types']} == {
1263
+ 'dataset': 2, 'model': 1, 'space': 1}
1264
+ assert {item['v']: item['n'] for item in facets['artifact_kinds']} == {
1265
+ 'recipe': 3, 'archive': 1}
1266
+ assert {item['v']: item['n'] for item in facets['base_images']} == {
1267
+ 'nvidia/cuda:12.8.0-runtime-ubuntu22.04': 1,
1268
+ 'rocm/pytorch:latest': 1,
1269
+ 'python:3.10-slim': 1,
1270
+ 'ubuntu:22.04': 1,
1271
+ }
1272
+ assert {item['v']: item['n'] for item in facets['accelerators']} == {
1273
+ 'nvidia': 2, 'amd': 1}
1274
+ assert {item['v']: item['n'] for item in facets['python_versions']} == {
1275
+ '3.11': 2, '3.12': 1, '3.10': 1}
1276
+ assert {item['v']: item['n'] for item in facets['frameworks']} == {
1277
+ 'pytorch': 3, 'fastapi': 1}
1278
+ assert {item['v']: item['n'] for item in facets['services']} == {
1279
+ 'api': 1, 'jupyter': 1, 'postgres': 1, 'worker': 1}
1280
+ assert {item['v']: item['n'] for item in facets['ports']} == {
1281
+ '7860': 2, '5432': 1, '8080': 1, '8888': 1}
1282
+ assert facets['has_compose'] == [
1283
+ {'v': True, 'n': 1}, {'v': False, 'n': 3}]
1284
+ assert facets['has_devcontainer'] == [
1285
+ {'v': True, 'n': 1}, {'v': False, 'n': 3}]
1286
+ assert facets['has_multistage'] == [
1287
+ {'v': True, 'n': 1}, {'v': False, 'n': 3}]