Spaces:
Running
Running
B-Sides app code (indexes load from dataset repos at boot)
Browse files- .DS_Store +0 -0
- .pytest_cache/.gitignore +2 -0
- .pytest_cache/CACHEDIR.TAG +4 -0
- .pytest_cache/README.md +8 -0
- .pytest_cache/v/cache/nodeids +117 -0
- Dockerfile +22 -0
- README.md +57 -6
- __pycache__/app.cpython-311.pyc +0 -0
- __pycache__/app.cpython-313.pyc +0 -0
- __pycache__/build_index.cpython-311.pyc +0 -0
- __pycache__/build_index.cpython-314.pyc +0 -0
- __pycache__/rebuild_index_repo.cpython-314.pyc +0 -0
- __pycache__/run_container_job.cpython-311.pyc +0 -0
- __pycache__/run_container_job.cpython-314.pyc +0 -0
- __pycache__/run_hf_job.cpython-311.pyc +0 -0
- app.py +265 -0
- bsides/.DS_Store +0 -0
- bsides/__init__.py +5 -0
- bsides/config.py +202 -0
- bsides/search_indexes.py +1003 -0
- requirements-dev.txt +5 -0
- requirements.txt +8 -0
- staff_picks.json +22 -0
- static/index.html +1217 -0
- tests/.DS_Store +0 -0
- tests/test_config.py +127 -0
- tests/test_runtime.py +874 -0
- tests/test_search_indexes.py +1287 -0
.DS_Store
ADDED
|
Binary file (10.2 kB). View file
|
|
|
.pytest_cache/.gitignore
ADDED
|
@@ -0,0 +1,2 @@
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Created by pytest automatically.
|
| 2 |
+
*
|
.pytest_cache/CACHEDIR.TAG
ADDED
|
@@ -0,0 +1,4 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
Signature: 8a477f597d28d172789f06886806bc55
|
| 2 |
+
# This file is a cache directory tag created by pytest.
|
| 3 |
+
# For information about cache directory tags, see:
|
| 4 |
+
# https://bford.info/cachedir/spec.html
|
.pytest_cache/README.md
ADDED
|
@@ -0,0 +1,8 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# pytest cache directory #
|
| 2 |
+
|
| 3 |
+
This directory contains data from the pytest's cache plugin,
|
| 4 |
+
which provides the `--lf` and `--ff` options, as well as the `cache` fixture.
|
| 5 |
+
|
| 6 |
+
**Do not** commit this to version control.
|
| 7 |
+
|
| 8 |
+
See [the docs](https://docs.pytest.org/en/stable/how-to/cache.html) for more information.
|
.pytest_cache/v/cache/nodeids
ADDED
|
@@ -0,0 +1,117 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
[
|
| 2 |
+
"tests/test_config.py::test_collection_validation_names_missing_credential[env0-HF_TOKEN]",
|
| 3 |
+
"tests/test_config.py::test_collection_validation_names_missing_credential[env1-OPENAI_API_KEY]",
|
| 4 |
+
"tests/test_config.py::test_crawl_end_is_an_explicit_inclusive_upper_boundary",
|
| 5 |
+
"tests/test_config.py::test_defaults_match_the_production_crawl_contract",
|
| 6 |
+
"tests/test_config.py::test_environment_overrides_are_parsed_without_storing_secret_values",
|
| 7 |
+
"tests/test_config.py::test_invalid_settings_are_rejected[env0-BSIDES_MODE]",
|
| 8 |
+
"tests/test_config.py::test_invalid_settings_are_rejected[env1-Hub request budget]",
|
| 9 |
+
"tests/test_config.py::test_invalid_settings_are_rejected[env2-Hub request budget]",
|
| 10 |
+
"tests/test_config.py::test_invalid_settings_are_rejected[env3-embedding dimensions]",
|
| 11 |
+
"tests/test_config.py::test_invalid_settings_are_rejected[env4-overlap]",
|
| 12 |
+
"tests/test_config.py::test_invalid_settings_are_rejected[env5-queue multiplier]",
|
| 13 |
+
"tests/test_config.py::test_invalid_settings_are_rejected[env6-embedding workers]",
|
| 14 |
+
"tests/test_config.py::test_invalid_settings_are_rejected[env7-embedding token rate]",
|
| 15 |
+
"tests/test_config.py::test_invalid_settings_are_rejected[env8-checkpoint interval]",
|
| 16 |
+
"tests/test_config.py::test_invalid_settings_are_rejected[env9-progress interval]",
|
| 17 |
+
"tests/test_config.py::test_search_mode_does_not_require_collection_credentials",
|
| 18 |
+
"tests/test_runtime.py::test_app_registers_docker_corpus_configuration",
|
| 19 |
+
"tests/test_runtime.py::test_app_registers_kernel_corpus_configuration",
|
| 20 |
+
"tests/test_runtime.py::test_docker_facets_and_search_use_container_response_shape",
|
| 21 |
+
"tests/test_runtime.py::test_docker_index_download_allows_only_published_search_files",
|
| 22 |
+
"tests/test_runtime.py::test_docker_semantic_search_without_a_key_is_unavailable_not_a_crash",
|
| 23 |
+
"tests/test_runtime.py::test_endpoints_503_until_the_index_is_ready",
|
| 24 |
+
"tests/test_runtime.py::test_facets_and_picks_still_answer",
|
| 25 |
+
"tests/test_runtime.py::test_failed_docker_load_does_not_affect_model_search",
|
| 26 |
+
"tests/test_runtime.py::test_failed_kernel_load_does_not_affect_model_search",
|
| 27 |
+
"tests/test_runtime.py::test_frontend_checks_model_exclusions_by_default",
|
| 28 |
+
"tests/test_runtime.py::test_frontend_defaults_docker_to_recipes_with_an_archive_toggle",
|
| 29 |
+
"tests/test_runtime.py::test_frontend_escapes_every_docker_card_value",
|
| 30 |
+
"tests/test_runtime.py::test_frontend_keeps_model_size_as_precise_billion_boxes",
|
| 31 |
+
"tests/test_runtime.py::test_frontend_links_docker_artifacts_to_valid_hugging_face_targets",
|
| 32 |
+
"tests/test_runtime.py::test_frontend_offers_all_three_corpora",
|
| 33 |
+
"tests/test_runtime.py::test_frontend_restores_default_exclusions_when_clearing_the_deck",
|
| 34 |
+
"tests/test_runtime.py::test_frontend_routes_every_corpus_through_one_url_builder",
|
| 35 |
+
"tests/test_runtime.py::test_frontend_tracks_docker_in_every_corpus_state_map",
|
| 36 |
+
"tests/test_runtime.py::test_kernel_facets_and_search_use_kernel_response_shape",
|
| 37 |
+
"tests/test_runtime.py::test_model_search_exclusions_are_explicit_and_api_compatible",
|
| 38 |
+
"tests/test_runtime.py::test_models_client_remains_the_oai_compatibility_shim",
|
| 39 |
+
"tests/test_runtime.py::test_reload_endpoint_hot_swaps_only_the_docker_corpus",
|
| 40 |
+
"tests/test_runtime.py::test_reload_endpoint_hot_swaps_only_the_kernel_corpus",
|
| 41 |
+
"tests/test_runtime.py::test_reload_of_an_unknown_corpus_is_a_404",
|
| 42 |
+
"tests/test_runtime.py::test_reload_with_a_wrong_token_is_403",
|
| 43 |
+
"tests/test_runtime.py::test_reload_with_an_empty_body_is_rejected",
|
| 44 |
+
"tests/test_runtime.py::test_reload_with_the_right_token_swaps_the_index",
|
| 45 |
+
"tests/test_runtime.py::test_reload_without_a_configured_token_is_503",
|
| 46 |
+
"tests/test_runtime.py::test_search_defaults_to_the_models_corpus",
|
| 47 |
+
"tests/test_runtime.py::test_search_index_download_excludes_crawler_staging",
|
| 48 |
+
"tests/test_runtime.py::test_search_query_uses_the_loaded_index_dimensions",
|
| 49 |
+
"tests/test_runtime.py::test_search_response_keeps_the_total_models_envelope",
|
| 50 |
+
"tests/test_runtime.py::test_semantic_search_without_a_key_is_unavailable_not_a_crash",
|
| 51 |
+
"tests/test_runtime.py::test_semantic_searches_use_their_assigned_corpus_clients",
|
| 52 |
+
"tests/test_runtime.py::test_unknown_corpus_is_a_404",
|
| 53 |
+
"tests/test_search_indexes.py::test_build_where_covers_every_model_filter",
|
| 54 |
+
"tests/test_search_indexes.py::test_build_where_empty_request_is_neutral",
|
| 55 |
+
"tests/test_search_indexes.py::test_build_where_executes_against_real_sqlite",
|
| 56 |
+
"tests/test_search_indexes.py::test_card_decodes_json_columns_and_rounds_score",
|
| 57 |
+
"tests/test_search_indexes.py::test_card_without_score_omits_it",
|
| 58 |
+
"tests/test_search_indexes.py::test_compute_facets_shape_matches_the_legacy_payload",
|
| 59 |
+
"tests/test_search_indexes.py::test_container_archive_rows_appear_when_artifact_kind_filter_is_absent",
|
| 60 |
+
"tests/test_search_indexes.py::test_container_card_decodes_json_arrays_and_preserves_empty_arrays",
|
| 61 |
+
"tests/test_search_indexes.py::test_container_defaults_to_requested_recipe_filter",
|
| 62 |
+
"tests/test_search_indexes.py::test_container_facets_count_scalars_arrays_and_booleans",
|
| 63 |
+
"tests/test_search_indexes.py::test_container_filters_are_anded_across_groups_and_ored_within_group",
|
| 64 |
+
"tests/test_search_indexes.py::test_container_query_text_is_preserved_for_semantic_search",
|
| 65 |
+
"tests/test_search_indexes.py::test_container_scalar_boolean_and_json_filters_execute_against_sqlite",
|
| 66 |
+
"tests/test_search_indexes.py::test_db_is_thread_local_readonly_and_generation_scoped",
|
| 67 |
+
"tests/test_search_indexes.py::test_download_index_requests_dataset_files_into_spec_dir",
|
| 68 |
+
"tests/test_search_indexes.py::test_embed_query_caps_input_at_max_chars",
|
| 69 |
+
"tests/test_search_indexes.py::test_embed_query_uses_spec_model_and_selected_matrix_dimensions",
|
| 70 |
+
"tests/test_search_indexes.py::test_embed_query_without_client_names_the_selected_credential",
|
| 71 |
+
"tests/test_search_indexes.py::test_embed_query_without_matrix_is_unavailable",
|
| 72 |
+
"tests/test_search_indexes.py::test_extract_constraints_empty_remainder_skips_embedding",
|
| 73 |
+
"tests/test_search_indexes.py::test_extract_constraints_never_loosens_explicit_filters",
|
| 74 |
+
"tests/test_search_indexes.py::test_extract_constraints_pulls_dates_params_downloads_and_flags",
|
| 75 |
+
"tests/test_search_indexes.py::test_failed_download_keeps_serving_the_previous_index",
|
| 76 |
+
"tests/test_search_indexes.py::test_kernel_adapter_ignores_legacy_model_and_unknown_filters",
|
| 77 |
+
"tests/test_search_indexes.py::test_kernel_array_filter_values_are_or_within_one_category",
|
| 78 |
+
"tests/test_search_indexes.py::test_kernel_card_decodes_arrays_and_returns_facts",
|
| 79 |
+
"tests/test_search_indexes.py::test_kernel_facets_count_members_of_json_arrays",
|
| 80 |
+
"tests/test_search_indexes.py::test_kernel_filters_execute_against_real_sqlite",
|
| 81 |
+
"tests/test_search_indexes.py::test_kernel_query_text_is_preserved_for_semantic_search",
|
| 82 |
+
"tests/test_search_indexes.py::test_kernel_structured_page_uses_kernel_envelope_contract",
|
| 83 |
+
"tests/test_search_indexes.py::test_load_accepts_empty_index",
|
| 84 |
+
"tests/test_search_indexes.py::test_load_isolates_downloader_failure",
|
| 85 |
+
"tests/test_search_indexes.py::test_load_pulls_dataset_when_files_are_missing",
|
| 86 |
+
"tests/test_search_indexes.py::test_load_rejects_dimension_mismatch",
|
| 87 |
+
"tests/test_search_indexes.py::test_load_rejects_matrix_rank_not_two",
|
| 88 |
+
"tests/test_search_indexes.py::test_load_rejects_non_finite_values",
|
| 89 |
+
"tests/test_search_indexes.py::test_load_rejects_row_count_mismatch",
|
| 90 |
+
"tests/test_search_indexes.py::test_load_rejects_rows_outside_matrix_bounds",
|
| 91 |
+
"tests/test_search_indexes.py::test_load_rejects_sqlite_that_fails_quick_check",
|
| 92 |
+
"tests/test_search_indexes.py::test_load_validates_marks_ready_and_computes_facets",
|
| 93 |
+
"tests/test_search_indexes.py::test_load_without_downloader_never_imports_hub_when_files_exist",
|
| 94 |
+
"tests/test_search_indexes.py::test_malformed_config_remains_searchable_during_quantization_exclusion",
|
| 95 |
+
"tests/test_search_indexes.py::test_model_adapter_exposes_spec_result_key_and_card_columns",
|
| 96 |
+
"tests/test_search_indexes.py::test_model_adapter_ignores_adapter_filters",
|
| 97 |
+
"tests/test_search_indexes.py::test_model_exclusions_are_neutral_by_default",
|
| 98 |
+
"tests/test_search_indexes.py::test_model_exclusions_match_family_evidence",
|
| 99 |
+
"tests/test_search_indexes.py::test_model_family_exclusion_matches_expanded_mistral_terms",
|
| 100 |
+
"tests/test_search_indexes.py::test_quantization_exclusion_requires_strong_evidence",
|
| 101 |
+
"tests/test_search_indexes.py::test_registry_isolates_a_broken_corpus_from_a_healthy_one",
|
| 102 |
+
"tests/test_search_indexes.py::test_registry_start_loading_runs_isolated_threads",
|
| 103 |
+
"tests/test_search_indexes.py::test_registry_unknown_key_raises_keyerror",
|
| 104 |
+
"tests/test_search_indexes.py::test_reload_does_not_skip_the_download_step",
|
| 105 |
+
"tests/test_search_indexes.py::test_reload_invalidates_connections_to_the_replaced_file",
|
| 106 |
+
"tests/test_search_indexes.py::test_reload_rejecting_a_bad_publish_keeps_serving_the_old_index",
|
| 107 |
+
"tests/test_search_indexes.py::test_reload_rejects_a_publish_whose_rows_do_not_match_the_matrix",
|
| 108 |
+
"tests/test_search_indexes.py::test_reload_swaps_in_the_freshly_published_index",
|
| 109 |
+
"tests/test_search_indexes.py::test_search_request_keeps_optional_adapter_filters",
|
| 110 |
+
"tests/test_search_indexes.py::test_semantic_rank_empty_candidates_skip_embedding",
|
| 111 |
+
"tests/test_search_indexes.py::test_semantic_rank_page_beyond_window_returns_total_but_no_cards",
|
| 112 |
+
"tests/test_search_indexes.py::test_semantic_rank_propagates_unavailable_embedding",
|
| 113 |
+
"tests/test_search_indexes.py::test_semantic_rank_scores_orders_and_paginates",
|
| 114 |
+
"tests/test_search_indexes.py::test_semantic_rank_uses_candidate_scoring_for_small_filtered_sets",
|
| 115 |
+
"tests/test_search_indexes.py::test_structured_page_orders_counts_and_paginates",
|
| 116 |
+
"tests/test_search_indexes.py::test_structured_page_unknown_sort_falls_back_to_newest"
|
| 117 |
+
]
|
Dockerfile
ADDED
|
@@ -0,0 +1,22 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
FROM python:3.12-slim
|
| 2 |
+
|
| 3 |
+
WORKDIR /app
|
| 4 |
+
ENV BSIDES_MODE=search
|
| 5 |
+
# git isn't in the slim base, but HF's Space builder injects a
|
| 6 |
+
# `git config --global user.email ...` step after our layers β it 127s without git.
|
| 7 |
+
RUN apt-get update && apt-get install -y --no-install-recommends git \
|
| 8 |
+
&& rm -rf /var/lib/apt/lists/*
|
| 9 |
+
COPY requirements.txt .
|
| 10 |
+
RUN pip install --no-cache-dir -r requirements.txt
|
| 11 |
+
|
| 12 |
+
# Retrieval only: the collector and its job scripts are not part of this image.
|
| 13 |
+
COPY app.py staff_picks.json ./
|
| 14 |
+
COPY bsides/__init__.py bsides/config.py bsides/search_indexes.py bsides/
|
| 15 |
+
COPY static static
|
| 16 |
+
|
| 17 |
+
# index is pulled from the b-sides-index dataset at boot (see app.py)
|
| 18 |
+
RUN mkdir -p /app/index && chmod 777 /app/index
|
| 19 |
+
RUN mkdir -p /tmp/bsides && chmod 777 /tmp/bsides
|
| 20 |
+
|
| 21 |
+
EXPOSE 7860
|
| 22 |
+
CMD ["uvicorn", "app:app", "--host", "0.0.0.0", "--port", "7860"]
|
README.md
CHANGED
|
@@ -1,11 +1,62 @@
|
|
| 1 |
---
|
| 2 |
-
title: B
|
| 3 |
-
emoji:
|
| 4 |
-
colorFrom:
|
| 5 |
-
colorTo:
|
| 6 |
sdk: docker
|
|
|
|
| 7 |
pinned: false
|
| 8 |
-
|
| 9 |
---
|
| 10 |
|
| 11 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
---
|
| 2 |
+
title: B-Sides V2
|
| 3 |
+
emoji: π
|
| 4 |
+
colorFrom: red
|
| 5 |
+
colorTo: yellow
|
| 6 |
sdk: docker
|
| 7 |
+
app_port: 7860
|
| 8 |
pinned: false
|
| 9 |
+
short_description: Building live to let the obscure breathe new life
|
| 10 |
---
|
| 11 |
|
| 12 |
+
# π B-Sides V2 β the full text-model record
|
| 13 |
+
|
| 14 |
+
B-Sides V2 is a fresh, searchable full-history pressing of Hugging Face text
|
| 15 |
+
models.
|
| 16 |
+
|
| 17 |
+
B-Sides index is more interesting than "models nobody uses." It maps the shadow
|
| 18 |
+
infrastructure of open ML: abandoned experiments, regional language work,
|
| 19 |
+
architecture probes, creative utilities, and research releases that conventional
|
| 20 |
+
popularity rankings erase.
|
| 21 |
+
|
| 22 |
+
The production collector walks both official Hub pipeline streams from newest
|
| 23 |
+
to oldest until they are exhausted:
|
| 24 |
+
|
| 25 |
+
- `text-generation`
|
| 26 |
+
- `text2text-generation`
|
| 27 |
+
|
| 28 |
+
It deduplicates full repository IDs. There are no family, popularity, download,
|
| 29 |
+
parameter, likes, license, gating, private-visibility, or source-content
|
| 30 |
+
exclusions: every discovered model receives a metadata embedding, and available
|
| 31 |
+
cards/configuration/source are added to it.
|
| 32 |
+
|
| 33 |
+
For every survivor, B-Sides fetches the root model card plus every Python file at
|
| 34 |
+
the discovered commit SHA, chunks them by source lines, and sends deterministic
|
| 35 |
+
requests to `text-embedding-3-small` through the OpenAI Batch API. The collector
|
| 36 |
+
checkpoints its SQLite state to
|
| 37 |
+
[`juiceb0xc0de/b-sides-v2-index`](https://huggingface.co/datasets/juiceb0xc0de/b-sides-v2-index),
|
| 38 |
+
resumes after restarts, and moves into overlapping tail scans after the initial
|
| 39 |
+
backfill.
|
| 40 |
+
|
| 41 |
+
All Hub traffic shares a conservative governor: at most 2,400 counted or
|
| 42 |
+
reserved requests per 300 seconds against the confirmed 3,000-request hard cap.
|
| 43 |
+
|
| 44 |
+
The existing search index remains online while this full-history pressing is
|
| 45 |
+
built. New search activation stays gated until coverage, chunk/vector alignment,
|
| 46 |
+
dimensions, finite values, and checksums validate.
|
| 47 |
+
|
| 48 |
+
Collector progress is available at `/api/crawl-status`.
|
| 49 |
+
|
| 50 |
+
## Runtime configuration
|
| 51 |
+
|
| 52 |
+
- `BSIDES_MODE=search|collect|both` (default: `both`)
|
| 53 |
+
- `BSIDES_DATASET` (default: `juiceb0xc0de/b-sides-v2-index`)
|
| 54 |
+
- `BSIDES_RUN_ID` (default: `v2`)
|
| 55 |
+
- `BSIDES_CRAWL_END` (optional ISO-8601 exclusive upper boundary)
|
| 56 |
+
- `BSIDES_HUB_REQUESTS_PER_5M` (default: `2400`, must stay below `3000`)
|
| 57 |
+
- `BSIDES_CHECKPOINT_EVERY_REPOS` (default: `2500`)
|
| 58 |
+
- `BSIDES_TAIL_INTERVAL_SECONDS` (default: `900`)
|
| 59 |
+
- `BSIDES_TAIL_OVERLAP_IDS` (default: `100`)
|
| 60 |
+
- Space secrets: `HF_TOKEN` and `OPENAI_API_KEY`
|
| 61 |
+
|
| 62 |
+
Pressed by [juiceb0xc0de](https://huggingface.co/juiceb0xc0de).
|
__pycache__/app.cpython-311.pyc
ADDED
|
Binary file (12.8 kB). View file
|
|
|
__pycache__/app.cpython-313.pyc
ADDED
|
Binary file (11.6 kB). View file
|
|
|
__pycache__/build_index.cpython-311.pyc
ADDED
|
Binary file (21.1 kB). View file
|
|
|
__pycache__/build_index.cpython-314.pyc
ADDED
|
Binary file (20.2 kB). View file
|
|
|
__pycache__/rebuild_index_repo.cpython-314.pyc
ADDED
|
Binary file (10.8 kB). View file
|
|
|
__pycache__/run_container_job.cpython-311.pyc
ADDED
|
Binary file (6.4 kB). View file
|
|
|
__pycache__/run_container_job.cpython-314.pyc
ADDED
|
Binary file (4.06 kB). View file
|
|
|
__pycache__/run_hf_job.cpython-311.pyc
ADDED
|
Binary file (12.2 kB). View file
|
|
|
app.py
ADDED
|
@@ -0,0 +1,265 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""
|
| 2 |
+
B-Sides β deep search for indie & vintage text-generation LLMs.
|
| 3 |
+
|
| 4 |
+
FastAPI backend. A thin HTTP layer over `bsides.search_indexes`: the registry
|
| 5 |
+
owns downloading, loading, validation, facets and ranking for every corpus
|
| 6 |
+
(models, kernels and Docker images). This module owns the wire shape and
|
| 7 |
+
nothing else.
|
| 8 |
+
|
| 9 |
+
Retrieval only. Collection lives in its own job repos: this Space reads the
|
| 10 |
+
published indexes and never crawls, so nothing here imports the collector.
|
| 11 |
+
|
| 12 |
+
Endpoints:
|
| 13 |
+
GET / static frontend
|
| 14 |
+
GET /api/facets filter options + counts for one corpus
|
| 15 |
+
POST /api/search semantic + structured search over one corpus
|
| 16 |
+
POST /api/reload token-gated hot-swap of a freshly published index
|
| 17 |
+
GET /api/picks the daily staff-pick spread (models corpus)
|
| 18 |
+
"""
|
| 19 |
+
|
| 20 |
+
import hmac
|
| 21 |
+
import os
|
| 22 |
+
import threading
|
| 23 |
+
from contextlib import asynccontextmanager
|
| 24 |
+
from pathlib import Path
|
| 25 |
+
|
| 26 |
+
import orjson
|
| 27 |
+
from fastapi import FastAPI, HTTPException
|
| 28 |
+
from fastapi.responses import FileResponse, Response
|
| 29 |
+
from openai import OpenAI
|
| 30 |
+
from pydantic import BaseModel
|
| 31 |
+
|
| 32 |
+
from bsides.config import Settings
|
| 33 |
+
from bsides.search_indexes import (
|
| 34 |
+
PAGE_SIZE,
|
| 35 |
+
ContainerAdapter,
|
| 36 |
+
IndexRegistry,
|
| 37 |
+
IndexSpec,
|
| 38 |
+
KernelAdapter,
|
| 39 |
+
ModelAdapter,
|
| 40 |
+
SearchReq,
|
| 41 |
+
UnavailableError,
|
| 42 |
+
semantic_rank,
|
| 43 |
+
structured_page,
|
| 44 |
+
)
|
| 45 |
+
|
| 46 |
+
BASE = Path(__file__).parent
|
| 47 |
+
INDEX_DIR = Path(os.environ.get('BSIDES_INDEX', BASE / 'index'))
|
| 48 |
+
INDEX_DATASET = os.environ.get('BSIDES_DATASET', 'juiceb0xc0de/b-sides-v2-index')
|
| 49 |
+
KERNEL_DATASET = os.environ.get(
|
| 50 |
+
'BSIDES_KERNEL_DATASET', 'juiceb0xc0de/b-sides-v2-kernels')
|
| 51 |
+
CONTAINER_DATASET = os.environ.get(
|
| 52 |
+
'BSIDES_CONTAINER_DATASET', 'juiceb0xc0de/b-sides-v2-containers')
|
| 53 |
+
|
| 54 |
+
EMBED_MODEL = 'text-embedding-3-small' # must match the corpus embedding model
|
| 55 |
+
MODELS = 'models' # default corpus for every endpoint
|
| 56 |
+
KERNELS = 'kernels'
|
| 57 |
+
DOCKER = 'docker'
|
| 58 |
+
QUERY_CREDENTIALS = {
|
| 59 |
+
MODELS: 'OPENAI_API_KEY',
|
| 60 |
+
KERNELS: 'OPENAI_KERNEL_API_KEY',
|
| 61 |
+
DOCKER: 'OPENAI_DOCKER_API_KEY',
|
| 62 |
+
}
|
| 63 |
+
|
| 64 |
+
# The published matrix width is a property of the crawl that produced it, so the
|
| 65 |
+
# spec's expected width comes from the same Settings the collector embeds with.
|
| 66 |
+
# It is a validation expectation only β the width actually sent to the embedding
|
| 67 |
+
# API is read off the loaded matrix (see search_indexes.embed_query).
|
| 68 |
+
EMBED_DIMENSIONS = Settings.from_env(os.environ).embedding_dimensions
|
| 69 |
+
|
| 70 |
+
|
| 71 |
+
@asynccontextmanager
|
| 72 |
+
async def _lifespan(_application: FastAPI):
|
| 73 |
+
_startup()
|
| 74 |
+
yield
|
| 75 |
+
|
| 76 |
+
|
| 77 |
+
app = FastAPI(title='B-Sides', lifespan=_lifespan)
|
| 78 |
+
|
| 79 |
+
|
| 80 |
+
def _openai_client(env_name: str):
|
| 81 |
+
key = os.environ.get(env_name)
|
| 82 |
+
return OpenAI(api_key=key) if key else None
|
| 83 |
+
|
| 84 |
+
|
| 85 |
+
query_clients = {
|
| 86 |
+
corpus: _openai_client(credential)
|
| 87 |
+
for corpus, credential in QUERY_CREDENTIALS.items()
|
| 88 |
+
}
|
| 89 |
+
oai = query_clients[MODELS]
|
| 90 |
+
|
| 91 |
+
# ββ corpora βββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 92 |
+
# Registration is import-time and cheap; the 1.4 GB matrix β and on a cold boot
|
| 93 |
+
# the ~800 MB index download β load in background threads kicked off by the
|
| 94 |
+
# lifespan startup. That is the whole point: uvicorn binds :7860 immediately, so
|
| 95 |
+
# HF's Space watchdog stops re-spawning duplicate workers while the matrix is
|
| 96 |
+
# still streaming off disk. Until an index is ready its endpoints 503.
|
| 97 |
+
MODEL_SPEC = IndexSpec(
|
| 98 |
+
key=MODELS,
|
| 99 |
+
dataset_id=INDEX_DATASET,
|
| 100 |
+
local_dir=INDEX_DIR,
|
| 101 |
+
table='models',
|
| 102 |
+
id_column='model_id',
|
| 103 |
+
embed_model=EMBED_MODEL,
|
| 104 |
+
dimensions=EMBED_DIMENSIONS,
|
| 105 |
+
)
|
| 106 |
+
KERNEL_SPEC = IndexSpec(
|
| 107 |
+
key=KERNELS,
|
| 108 |
+
dataset_id=KERNEL_DATASET,
|
| 109 |
+
local_dir=INDEX_DIR / KERNELS,
|
| 110 |
+
table='kernels',
|
| 111 |
+
id_column='repo_id',
|
| 112 |
+
embed_model=EMBED_MODEL,
|
| 113 |
+
dimensions=256,
|
| 114 |
+
)
|
| 115 |
+
DOCKER_SPEC = IndexSpec(
|
| 116 |
+
key=DOCKER,
|
| 117 |
+
dataset_id=CONTAINER_DATASET,
|
| 118 |
+
local_dir=INDEX_DIR / DOCKER,
|
| 119 |
+
table='containers',
|
| 120 |
+
id_column='repo_id',
|
| 121 |
+
embed_model=EMBED_MODEL,
|
| 122 |
+
dimensions=256,
|
| 123 |
+
)
|
| 124 |
+
|
| 125 |
+
registry = IndexRegistry()
|
| 126 |
+
registry.register(ModelAdapter(MODEL_SPEC))
|
| 127 |
+
registry.register(KernelAdapter(KERNEL_SPEC))
|
| 128 |
+
registry.register(ContainerAdapter(DOCKER_SPEC))
|
| 129 |
+
|
| 130 |
+
|
| 131 |
+
def _ready_index(corpus: str):
|
| 132 |
+
"""Resolve a corpus key to a loaded index, or fail the way HTTP expects."""
|
| 133 |
+
try:
|
| 134 |
+
index = registry.index(corpus)
|
| 135 |
+
except KeyError:
|
| 136 |
+
raise HTTPException(404, f'unknown corpus: {corpus}') from None
|
| 137 |
+
if not index.ready.is_set():
|
| 138 |
+
# `error` set means load() failed and recorded why β that state is
|
| 139 |
+
# permanent until the artifact is fixed, so "retry in a moment" would
|
| 140 |
+
# be a lie. (Reload failures keep ready set and never land here.)
|
| 141 |
+
if index.error is not None:
|
| 142 |
+
raise HTTPException(
|
| 143 |
+
503, f'index failed to load: {index.error}')
|
| 144 |
+
raise HTTPException(503, 'index still loading β retry in a moment')
|
| 145 |
+
return index
|
| 146 |
+
|
| 147 |
+
|
| 148 |
+
def _startup():
|
| 149 |
+
# bind first, load in the background β see the note above MODEL_SPEC.
|
| 150 |
+
registry.start_loading()
|
| 151 |
+
print('startup: :7860 binding while the index loads in the background',
|
| 152 |
+
flush=True)
|
| 153 |
+
|
| 154 |
+
|
| 155 |
+
# ββ search ββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 156 |
+
@app.post('/api/search')
|
| 157 |
+
def search(r: SearchReq, corpus: str = MODELS):
|
| 158 |
+
index = _ready_index(corpus)
|
| 159 |
+
adapter = index.adapter
|
| 160 |
+
# Fill the facet slots the user left empty from the query text, then filter.
|
| 161 |
+
# Order matters: extraction mutates the request that build_where reads.
|
| 162 |
+
sem_q = adapter.extract_constraints(r)
|
| 163 |
+
where, args = adapter.build_where(r)
|
| 164 |
+
try:
|
| 165 |
+
if sem_q and r.sort == 'relevance':
|
| 166 |
+
total, cards = semantic_rank(
|
| 167 |
+
index.db(), index, adapter, where=where, args=args,
|
| 168 |
+
sem_q=sem_q, page=r.page, page_size=PAGE_SIZE,
|
| 169 |
+
embed_client=query_clients[corpus],
|
| 170 |
+
credential_name=QUERY_CREDENTIALS[corpus])
|
| 171 |
+
else:
|
| 172 |
+
total, cards = structured_page(
|
| 173 |
+
index.db(), index, adapter, where=where, args=args,
|
| 174 |
+
sort=r.sort, page=r.page, page_size=PAGE_SIZE)
|
| 175 |
+
except UnavailableError as e:
|
| 176 |
+
raise HTTPException(503, str(e)) from e
|
| 177 |
+
return Response(orjson.dumps({'total': total, adapter.result_key: cards}),
|
| 178 |
+
media_type='application/json')
|
| 179 |
+
|
| 180 |
+
|
| 181 |
+
@app.get('/api/facets')
|
| 182 |
+
def facets(corpus: str = MODELS):
|
| 183 |
+
return Response(orjson.dumps(_ready_index(corpus).facets),
|
| 184 |
+
media_type='application/json')
|
| 185 |
+
|
| 186 |
+
|
| 187 |
+
# ββ reload ββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 188 |
+
# publish_loop.py POSTs here after every publish so the Space picks up the new
|
| 189 |
+
# index without a restart. The token gate matters: a reload checks large index
|
| 190 |
+
# artifacts on the Hub and can transfer changed files, so an open endpoint would
|
| 191 |
+
# let anyone burn our bandwidth on demand.
|
| 192 |
+
RELOAD_TOKEN = os.environ.get('BSIDES_RELOAD_TOKEN', '')
|
| 193 |
+
_reload_lock = threading.Lock()
|
| 194 |
+
|
| 195 |
+
|
| 196 |
+
class ReloadReq(BaseModel):
|
| 197 |
+
token: str = ''
|
| 198 |
+
|
| 199 |
+
|
| 200 |
+
@app.post('/api/reload')
|
| 201 |
+
def reload_corpus(body: ReloadReq, corpus: str = MODELS):
|
| 202 |
+
"""Kick off a hot-swap and answer immediately.
|
| 203 |
+
|
| 204 |
+
The download is the slow part and the caller times out at 60 s, so the
|
| 205 |
+
swap runs on a background thread; `LoadedIndex.reload` keeps the current
|
| 206 |
+
index serving until the new one has validated.
|
| 207 |
+
"""
|
| 208 |
+
if not RELOAD_TOKEN:
|
| 209 |
+
raise HTTPException(
|
| 210 |
+
503, 'reload not configured: set BSIDES_RELOAD_TOKEN')
|
| 211 |
+
if not hmac.compare_digest(body.token, RELOAD_TOKEN):
|
| 212 |
+
raise HTTPException(403, 'reload token does not match')
|
| 213 |
+
try:
|
| 214 |
+
index = registry.index(corpus)
|
| 215 |
+
except KeyError:
|
| 216 |
+
raise HTTPException(404, f'unknown corpus: {corpus}') from None
|
| 217 |
+
# Non-blocking: a publish that overlaps the previous one is told to wait
|
| 218 |
+
# rather than racing a second download into the same directory.
|
| 219 |
+
if not _reload_lock.acquire(blocking=False):
|
| 220 |
+
raise HTTPException(409, 'a reload is already running')
|
| 221 |
+
|
| 222 |
+
def run() -> None:
|
| 223 |
+
try:
|
| 224 |
+
index.reload(registry.downloader)
|
| 225 |
+
finally:
|
| 226 |
+
_reload_lock.release()
|
| 227 |
+
|
| 228 |
+
threading.Thread(target=run, name=f'bsides-reload-{corpus}',
|
| 229 |
+
daemon=True).start()
|
| 230 |
+
return {'status': 'reload started', 'corpus': corpus}
|
| 231 |
+
|
| 232 |
+
|
| 233 |
+
SPREAD_SIZE = 4
|
| 234 |
+
|
| 235 |
+
@app.get('/api/picks')
|
| 236 |
+
def picks():
|
| 237 |
+
"""The daily spread: SPREAD_SIZE cards drawn from the staff_picks.json pool,
|
| 238 |
+
rotated deterministically by UTC date β no state, fresh every midnight."""
|
| 239 |
+
index = _ready_index(MODELS)
|
| 240 |
+
adapter = index.adapter
|
| 241 |
+
picks_path = BASE / 'staff_picks.json'
|
| 242 |
+
if not picks_path.exists():
|
| 243 |
+
return Response(orjson.dumps({'picks': []}), media_type='application/json')
|
| 244 |
+
pool = orjson.loads(picks_path.read_bytes())
|
| 245 |
+
if len(pool) > SPREAD_SIZE:
|
| 246 |
+
import datetime, random
|
| 247 |
+
today = datetime.datetime.now(datetime.timezone.utc).strftime('%Y-%m-%d')
|
| 248 |
+
pool = random.Random(today).sample(pool, SPREAD_SIZE)
|
| 249 |
+
c = index.db()
|
| 250 |
+
out = []
|
| 251 |
+
for e in pool:
|
| 252 |
+
row = c.execute(
|
| 253 |
+
f'SELECT {adapter.card_columns} FROM {index.spec.table} '
|
| 254 |
+
f'WHERE {index.spec.id_column} = ?', (e['model_id'],)).fetchone()
|
| 255 |
+
if row:
|
| 256 |
+
d = adapter.card(row)
|
| 257 |
+
d['note'] = e.get('note', '')
|
| 258 |
+
d['picked_by'] = e.get('picked_by', 'staff')
|
| 259 |
+
out.append(d)
|
| 260 |
+
return Response(orjson.dumps({'picks': out}), media_type='application/json')
|
| 261 |
+
|
| 262 |
+
|
| 263 |
+
@app.get('/')
|
| 264 |
+
def home():
|
| 265 |
+
return FileResponse(BASE / 'static' / 'index.html')
|
bsides/.DS_Store
ADDED
|
Binary file (6.15 kB). View file
|
|
|
bsides/__init__.py
ADDED
|
@@ -0,0 +1,5 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""B-Sides collection and semantic-index pipeline."""
|
| 2 |
+
|
| 3 |
+
__all__ = ["__version__"]
|
| 4 |
+
|
| 5 |
+
__version__ = "0.1.0"
|
bsides/config.py
ADDED
|
@@ -0,0 +1,202 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Validated runtime settings for the B-Sides collector."""
|
| 2 |
+
|
| 3 |
+
from __future__ import annotations
|
| 4 |
+
|
| 5 |
+
from dataclasses import dataclass
|
| 6 |
+
from pathlib import Path
|
| 7 |
+
from typing import Mapping
|
| 8 |
+
|
| 9 |
+
|
| 10 |
+
PIPELINES = (
|
| 11 |
+
"text-generation",
|
| 12 |
+
"text2text-generation",
|
| 13 |
+
)
|
| 14 |
+
# V2 maps every model in its official text streams. Retained as an empty
|
| 15 |
+
# compatibility export for existing downstream sample tooling.
|
| 16 |
+
BLACKLIST: tuple[str, ...] = ()
|
| 17 |
+
|
| 18 |
+
# Listing query parameter per pipeline. The Hub's pipeline_tag=listings for
|
| 19 |
+
# text2text-generation began answering HTTP 200 with an empty page (confirmed
|
| 20 |
+
# 2026-08-30 in the b-sides-v2-index-2025 audit): the crawler read that as
|
| 21 |
+
# stream exhaustion and dropped ~8,300 models. filter= does a tag match and
|
| 22 |
+
# still enumerates the stream; the extra dual-tagged rows land under
|
| 23 |
+
# text-generation too and dedupe on repo id.
|
| 24 |
+
LISTING_PARAMS = {
|
| 25 |
+
"text-generation": "pipeline_tag",
|
| 26 |
+
"text2text-generation": "filter",
|
| 27 |
+
}
|
| 28 |
+
|
| 29 |
+
|
| 30 |
+
def _int(env: Mapping[str, str], name: str, default: int) -> int:
|
| 31 |
+
try:
|
| 32 |
+
return int(env.get(name, str(default)))
|
| 33 |
+
except ValueError as exc:
|
| 34 |
+
raise ValueError(f"{name} must be an integer") from exc
|
| 35 |
+
|
| 36 |
+
|
| 37 |
+
def _float(env: Mapping[str, str], name: str, default: float) -> float:
|
| 38 |
+
try:
|
| 39 |
+
return float(env.get(name, str(default)))
|
| 40 |
+
except ValueError as exc:
|
| 41 |
+
raise ValueError(f"{name} must be a number") from exc
|
| 42 |
+
|
| 43 |
+
|
| 44 |
+
@dataclass(frozen=True, slots=True)
|
| 45 |
+
class Settings:
|
| 46 |
+
pipelines: tuple[str, ...] = PIPELINES
|
| 47 |
+
blacklist: tuple[str, ...] = BLACKLIST
|
| 48 |
+
mode: str = "both"
|
| 49 |
+
dataset_repo_id: str = "juiceb0xc0de/b-sides-v2-index"
|
| 50 |
+
run_id: str = "v2"
|
| 51 |
+
crawl_year: int | None = None
|
| 52 |
+
crawl_end: str | None = None
|
| 53 |
+
state_dir: Path = Path("/tmp/bsides")
|
| 54 |
+
hub_requests_per_window: int = 2_400
|
| 55 |
+
# Zero observes resolver traffic without proactively throttling it. Public
|
| 56 |
+
# model file fetches are not consuming the account's resolver allowance;
|
| 57 |
+
# real 429 responses are retried by HubGateway instead.
|
| 58 |
+
hub_resolver_requests_per_window: int = 0
|
| 59 |
+
hub_window_seconds: float = 300.0
|
| 60 |
+
# Fetching is latency-bound, so admitted repositories resolve concurrently.
|
| 61 |
+
fetch_workers: int = 24
|
| 62 |
+
fetch_queue_multiplier: int = 4
|
| 63 |
+
embed_workers: int = 8
|
| 64 |
+
checkpoint_interval_seconds: float = 1_800.0
|
| 65 |
+
progress_interval_seconds: float = 10.0
|
| 66 |
+
embedding_model: str = "text-embedding-3-small"
|
| 67 |
+
embedding_dimensions: int = 256
|
| 68 |
+
embedding_tokens_per_minute: int = 4_500_000
|
| 69 |
+
chunk_target_tokens: int = 1_200
|
| 70 |
+
chunk_overlap_tokens: int = 150
|
| 71 |
+
max_file_bytes: int = 2 * 1024 * 1024
|
| 72 |
+
max_repo_bytes: int = 16 * 1024 * 1024
|
| 73 |
+
checkpoint_every_repos: int = 2_500
|
| 74 |
+
tail_interval_seconds: float = 900.0
|
| 75 |
+
tail_overlap_ids: int = 100
|
| 76 |
+
lease_ttl_seconds: float = 1_800.0
|
| 77 |
+
hf_token_present: bool = False
|
| 78 |
+
openai_api_key_present: bool = False
|
| 79 |
+
|
| 80 |
+
@classmethod
|
| 81 |
+
def from_env(cls, env: Mapping[str, str]) -> "Settings":
|
| 82 |
+
settings = cls(
|
| 83 |
+
mode=env.get("BSIDES_MODE", "both").strip().lower(),
|
| 84 |
+
dataset_repo_id=env.get(
|
| 85 |
+
"BSIDES_DATASET", "juiceb0xc0de/b-sides-v2-index"
|
| 86 |
+
).strip(),
|
| 87 |
+
run_id=env.get("BSIDES_RUN_ID", "v2").strip(),
|
| 88 |
+
crawl_year=(
|
| 89 |
+
_int(env, "BSIDES_YEAR", 0) if env.get("BSIDES_YEAR", "").strip() else None
|
| 90 |
+
),
|
| 91 |
+
crawl_end=env.get("BSIDES_CRAWL_END", "").strip() or None,
|
| 92 |
+
state_dir=Path(env.get("BSIDES_STATE_DIR", "/tmp/bsides")),
|
| 93 |
+
hub_requests_per_window=_int(
|
| 94 |
+
env, "BSIDES_HUB_REQUESTS_PER_5M", 2_400
|
| 95 |
+
),
|
| 96 |
+
hub_resolver_requests_per_window=_int(
|
| 97 |
+
env, "BSIDES_HUB_RESOLVER_REQUESTS_PER_5M", 0
|
| 98 |
+
),
|
| 99 |
+
fetch_workers=_int(env, "BSIDES_FETCH_WORKERS", 24),
|
| 100 |
+
fetch_queue_multiplier=_int(
|
| 101 |
+
env, "BSIDES_FETCH_QUEUE_MULTIPLIER", 4
|
| 102 |
+
),
|
| 103 |
+
embed_workers=_int(env, "BSIDES_EMBED_WORKERS", 8),
|
| 104 |
+
checkpoint_interval_seconds=_float(
|
| 105 |
+
env, "BSIDES_CHECKPOINT_INTERVAL_SECONDS", 1_800.0
|
| 106 |
+
),
|
| 107 |
+
progress_interval_seconds=_float(
|
| 108 |
+
env, "BSIDES_PROGRESS_INTERVAL_SECONDS", 10.0
|
| 109 |
+
),
|
| 110 |
+
embedding_dimensions=_int(
|
| 111 |
+
env, "BSIDES_EMBEDDING_DIMENSIONS", 256
|
| 112 |
+
),
|
| 113 |
+
embedding_tokens_per_minute=_int(
|
| 114 |
+
env, "BSIDES_EMBEDDING_TPM", 4_500_000
|
| 115 |
+
),
|
| 116 |
+
chunk_target_tokens=_int(
|
| 117 |
+
env, "BSIDES_CHUNK_TARGET_TOKENS", 1_200
|
| 118 |
+
),
|
| 119 |
+
chunk_overlap_tokens=_int(
|
| 120 |
+
env, "BSIDES_CHUNK_OVERLAP_TOKENS", 150
|
| 121 |
+
),
|
| 122 |
+
max_file_bytes=_int(env, "BSIDES_MAX_FILE_BYTES", 2 * 1024 * 1024),
|
| 123 |
+
max_repo_bytes=_int(
|
| 124 |
+
env, "BSIDES_MAX_REPO_BYTES", 16 * 1024 * 1024
|
| 125 |
+
),
|
| 126 |
+
checkpoint_every_repos=_int(
|
| 127 |
+
env, "BSIDES_CHECKPOINT_EVERY_REPOS", 2_500
|
| 128 |
+
),
|
| 129 |
+
tail_interval_seconds=_float(
|
| 130 |
+
env, "BSIDES_TAIL_INTERVAL_SECONDS", 900.0
|
| 131 |
+
),
|
| 132 |
+
tail_overlap_ids=_int(env, "BSIDES_TAIL_OVERLAP_IDS", 100),
|
| 133 |
+
lease_ttl_seconds=_float(
|
| 134 |
+
env, "BSIDES_LEASE_TTL_SECONDS", 1_800.0
|
| 135 |
+
),
|
| 136 |
+
hf_token_present=bool(env.get("HF_TOKEN", "").strip()),
|
| 137 |
+
openai_api_key_present=bool(env.get("OPENAI_API_KEY", "").strip()),
|
| 138 |
+
)
|
| 139 |
+
settings._validate()
|
| 140 |
+
return settings
|
| 141 |
+
|
| 142 |
+
def _validate(self) -> None:
|
| 143 |
+
if self.mode not in {"search", "collect", "both"}:
|
| 144 |
+
raise ValueError("BSIDES_MODE must be search, collect, or both")
|
| 145 |
+
if not 1 <= self.hub_requests_per_window < 3_000:
|
| 146 |
+
raise ValueError("Hub request budget must be between 1 and 2999")
|
| 147 |
+
if not 0 <= self.hub_resolver_requests_per_window < 20_000:
|
| 148 |
+
raise ValueError(
|
| 149 |
+
"Resolver budget must be between 0 and 19999"
|
| 150 |
+
)
|
| 151 |
+
if not 1 <= self.fetch_workers <= 64:
|
| 152 |
+
raise ValueError("fetch workers must be between 1 and 64")
|
| 153 |
+
if self.fetch_queue_multiplier <= 0:
|
| 154 |
+
raise ValueError("fetch queue multiplier must be positive")
|
| 155 |
+
if self.embed_workers <= 0:
|
| 156 |
+
raise ValueError("embedding workers must be positive")
|
| 157 |
+
if self.checkpoint_interval_seconds <= 0:
|
| 158 |
+
raise ValueError("checkpoint interval must be positive")
|
| 159 |
+
if self.progress_interval_seconds <= 0:
|
| 160 |
+
raise ValueError("progress interval must be positive")
|
| 161 |
+
if self.embedding_dimensions <= 0:
|
| 162 |
+
raise ValueError("embedding dimensions must be positive")
|
| 163 |
+
if self.embedding_tokens_per_minute <= 0:
|
| 164 |
+
raise ValueError("embedding token rate must be positive")
|
| 165 |
+
if self.chunk_target_tokens <= 0:
|
| 166 |
+
raise ValueError("chunk target must be positive")
|
| 167 |
+
if not 0 <= self.chunk_overlap_tokens < self.chunk_target_tokens:
|
| 168 |
+
raise ValueError("chunk overlap must be smaller than the target")
|
| 169 |
+
if self.max_file_bytes <= 0 or self.max_repo_bytes < self.max_file_bytes:
|
| 170 |
+
raise ValueError("artifact byte limits are inconsistent")
|
| 171 |
+
if self.checkpoint_every_repos <= 0:
|
| 172 |
+
raise ValueError("checkpoint interval must be positive")
|
| 173 |
+
if self.tail_interval_seconds <= 0:
|
| 174 |
+
raise ValueError("tail interval must be positive")
|
| 175 |
+
if self.tail_overlap_ids <= 0:
|
| 176 |
+
raise ValueError("tail overlap must be positive")
|
| 177 |
+
if self.lease_ttl_seconds <= 0:
|
| 178 |
+
raise ValueError("lease TTL must be positive")
|
| 179 |
+
if not self.dataset_repo_id:
|
| 180 |
+
raise ValueError("BSIDES_DATASET must not be empty")
|
| 181 |
+
if not self.run_id:
|
| 182 |
+
raise ValueError("BSIDES_RUN_ID must not be empty")
|
| 183 |
+
if self.crawl_year is not None and not 2000 <= self.crawl_year <= 9999:
|
| 184 |
+
raise ValueError("BSIDES_YEAR must be between 2000 and 9999")
|
| 185 |
+
if self.crawl_end is not None:
|
| 186 |
+
from datetime import datetime
|
| 187 |
+
|
| 188 |
+
try:
|
| 189 |
+
datetime.fromisoformat(self.crawl_end.replace("Z", "+00:00"))
|
| 190 |
+
except ValueError as exc:
|
| 191 |
+
raise ValueError("BSIDES_CRAWL_END must be an ISO-8601 timestamp") from exc
|
| 192 |
+
|
| 193 |
+
def validate_for_collection(self) -> None:
|
| 194 |
+
if self.mode == "search":
|
| 195 |
+
return
|
| 196 |
+
missing = []
|
| 197 |
+
if not self.hf_token_present:
|
| 198 |
+
missing.append("HF_TOKEN")
|
| 199 |
+
if not self.openai_api_key_present:
|
| 200 |
+
missing.append("OPENAI_API_KEY")
|
| 201 |
+
if missing:
|
| 202 |
+
raise ValueError("missing collection credentials: " + ", ".join(missing))
|
bsides/search_indexes.py
ADDED
|
@@ -0,0 +1,1003 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Reusable multi-corpus search index core.
|
| 2 |
+
|
| 3 |
+
A corpus (models today; kernels and Docker images next) is one normalized
|
| 4 |
+
embedding matrix plus one SQLite metadata table whose `row` column aligns
|
| 5 |
+
1:1 with matrix rows. This module owns everything schema-agnostic:
|
| 6 |
+
|
| 7 |
+
IndexSpec static corpus definition (dataset, table, embedding)
|
| 8 |
+
LoadedIndex runtime state: matrix, readiness, errors, connections
|
| 9 |
+
IndexRegistry registration, isolated loading, per-index status
|
| 10 |
+
embed_query query embedding against one index's model/dimensions
|
| 11 |
+
semantic_rank candidate rows β embed β score β top-k page
|
| 12 |
+
structured_page COUNT + ORDER BY paging without embeddings
|
| 13 |
+
|
| 14 |
+
Schema knowledge lives in CorpusAdapter implementations β ModelAdapter
|
| 15 |
+
reproduces today's model behavior verbatim. app.py stays a thin HTTP
|
| 16 |
+
layer over the registry.
|
| 17 |
+
|
| 18 |
+
Loader contract: `LoadedIndex.load` never raises. A corpus that fails to
|
| 19 |
+
download or validate records its error and stays not-ready; the other
|
| 20 |
+
corpora keep serving.
|
| 21 |
+
"""
|
| 22 |
+
|
| 23 |
+
from __future__ import annotations
|
| 24 |
+
|
| 25 |
+
import re
|
| 26 |
+
import sqlite3
|
| 27 |
+
import threading
|
| 28 |
+
from dataclasses import dataclass
|
| 29 |
+
from pathlib import Path
|
| 30 |
+
from typing import Protocol
|
| 31 |
+
|
| 32 |
+
import numpy as np
|
| 33 |
+
import orjson
|
| 34 |
+
from pydantic import BaseModel, Field
|
| 35 |
+
|
| 36 |
+
MATRIX_NAME = 'embeddings.f16.npy'
|
| 37 |
+
SQLITE_NAME = 'meta.sqlite'
|
| 38 |
+
DEFAULT_FILES = (MATRIX_NAME, SQLITE_NAME)
|
| 39 |
+
|
| 40 |
+
MAX_QUERY_CHARS = 300
|
| 41 |
+
PAGE_SIZE = 24
|
| 42 |
+
|
| 43 |
+
|
| 44 |
+
class UnavailableError(RuntimeError):
|
| 45 |
+
"""A search dependency is missing (embedding client, loaded index).
|
| 46 |
+
|
| 47 |
+
Raised from the core; the HTTP layer maps it to 503 with the message.
|
| 48 |
+
"""
|
| 49 |
+
|
| 50 |
+
|
| 51 |
+
@dataclass(frozen=True)
|
| 52 |
+
class IndexSpec:
|
| 53 |
+
"""Static definition of one searchable corpus."""
|
| 54 |
+
key: str # registry key: 'models' | 'kernels' | 'docker'
|
| 55 |
+
dataset_id: str # HF dataset repo publishing the index files
|
| 56 |
+
local_dir: Path # where the corpus files live on disk
|
| 57 |
+
table: str # SQLite table holding the metadata rows
|
| 58 |
+
id_column: str # human id column: 'model_id' | 'repo_id' | ...
|
| 59 |
+
embed_model: str # embedding model used for queries
|
| 60 |
+
dimensions: int # expected vector width; validated at load
|
| 61 |
+
files: tuple[str, ...] = DEFAULT_FILES
|
| 62 |
+
|
| 63 |
+
|
| 64 |
+
|
| 65 |
+
|
| 66 |
+
def _validate_matrix(spec: IndexSpec, matrix: np.ndarray) -> None:
|
| 67 |
+
if matrix.ndim != 2:
|
| 68 |
+
raise ValueError(
|
| 69 |
+
f'{spec.key}: matrix rank {matrix.ndim} != 2')
|
| 70 |
+
if matrix.shape[1] != spec.dimensions:
|
| 71 |
+
raise ValueError(
|
| 72 |
+
f'{spec.key}: matrix has {matrix.shape[1]} dimensions, '
|
| 73 |
+
f'spec expects {spec.dimensions}')
|
| 74 |
+
if not np.isfinite(matrix).all():
|
| 75 |
+
raise ValueError(f'{spec.key}: matrix contains non-finite values')
|
| 76 |
+
|
| 77 |
+
|
| 78 |
+
def _validate_sqlite(spec: IndexSpec, conn: sqlite3.Connection,
|
| 79 |
+
matrix_rows: int) -> None:
|
| 80 |
+
quick = conn.execute('PRAGMA quick_check').fetchone()[0]
|
| 81 |
+
if quick != 'ok':
|
| 82 |
+
raise ValueError(f'{spec.key}: sqlite quick_check failed: {quick}')
|
| 83 |
+
count = conn.execute(
|
| 84 |
+
f'SELECT COUNT(*) FROM {spec.table}').fetchone()[0]
|
| 85 |
+
if count != matrix_rows:
|
| 86 |
+
raise ValueError(
|
| 87 |
+
f'{spec.key}: {count} rows in {spec.table} vs '
|
| 88 |
+
f'{matrix_rows} matrix rows')
|
| 89 |
+
if count:
|
| 90 |
+
lo, hi = conn.execute(
|
| 91 |
+
f'SELECT MIN(row), MAX(row) FROM {spec.table}').fetchone()
|
| 92 |
+
if lo < 0 or hi >= matrix_rows:
|
| 93 |
+
raise ValueError(
|
| 94 |
+
f'{spec.key}: row ids {lo}..{hi} outside matrix bounds '
|
| 95 |
+
f'0..{matrix_rows - 1}')
|
| 96 |
+
|
| 97 |
+
|
| 98 |
+
class LoadedIndex:
|
| 99 |
+
"""Runtime state for one corpus: matrix, readiness, connections."""
|
| 100 |
+
|
| 101 |
+
def __init__(self, spec: IndexSpec, adapter=None) -> None:
|
| 102 |
+
self.spec = spec
|
| 103 |
+
self.adapter = adapter
|
| 104 |
+
self.matrix: np.ndarray | None = None
|
| 105 |
+
self.rows: int = 0
|
| 106 |
+
self.ready = threading.Event()
|
| 107 |
+
self.error: str | None = None
|
| 108 |
+
self.facets: dict = {}
|
| 109 |
+
self.generation = 0
|
| 110 |
+
self._local = threading.local()
|
| 111 |
+
|
| 112 |
+
def db(self) -> sqlite3.Connection:
|
| 113 |
+
"""Thread-local read-only connection, reopened after a swap."""
|
| 114 |
+
conn = getattr(self._local, 'conn', None)
|
| 115 |
+
if conn is not None and \
|
| 116 |
+
getattr(self._local, 'generation', -1) != self.generation:
|
| 117 |
+
# A reload swapped meta.sqlite underneath us; this connection
|
| 118 |
+
# still points at the replaced file.
|
| 119 |
+
conn.close()
|
| 120 |
+
conn = None
|
| 121 |
+
if conn is None:
|
| 122 |
+
conn = sqlite3.connect(
|
| 123 |
+
f'file:{self.spec.local_dir / SQLITE_NAME}?mode=ro', uri=True)
|
| 124 |
+
conn.row_factory = sqlite3.Row
|
| 125 |
+
self._local.conn = conn
|
| 126 |
+
self._local.generation = self.generation
|
| 127 |
+
return conn
|
| 128 |
+
|
| 129 |
+
def invalidate_connections(self) -> None:
|
| 130 |
+
"""Bump the generation so every thread reopens on next db()."""
|
| 131 |
+
self.generation += 1
|
| 132 |
+
|
| 133 |
+
def load(self, downloader=None) -> None:
|
| 134 |
+
"""Download-if-missing, load, validate, then flip ready.
|
| 135 |
+
|
| 136 |
+
Never raises: failure is recorded in `error` and `ready` stays
|
| 137 |
+
unset so this corpus 503s while the others keep serving.
|
| 138 |
+
Safe to call again after a failure: prior state is dropped up
|
| 139 |
+
front, so a retry can't leave ready/error disagreeing or a stale
|
| 140 |
+
matrix lingering behind the new attempt.
|
| 141 |
+
"""
|
| 142 |
+
self.ready.clear()
|
| 143 |
+
self.error = None
|
| 144 |
+
self.matrix = None
|
| 145 |
+
self.rows = 0
|
| 146 |
+
self.facets = {}
|
| 147 |
+
try:
|
| 148 |
+
if not (self.spec.local_dir / MATRIX_NAME).exists():
|
| 149 |
+
if downloader is None:
|
| 150 |
+
from huggingface_hub import snapshot_download
|
| 151 |
+
downloader = snapshot_download
|
| 152 |
+
download_index(downloader, self.spec)
|
| 153 |
+
matrix = np.load(
|
| 154 |
+
self.spec.local_dir / MATRIX_NAME).astype(np.float32)
|
| 155 |
+
_validate_matrix(self.spec, matrix)
|
| 156 |
+
check = sqlite3.connect(
|
| 157 |
+
f'file:{self.spec.local_dir / SQLITE_NAME}?mode=ro', uri=True)
|
| 158 |
+
try:
|
| 159 |
+
_validate_sqlite(self.spec, check, matrix_rows=matrix.shape[0])
|
| 160 |
+
finally:
|
| 161 |
+
check.close()
|
| 162 |
+
self.matrix = matrix
|
| 163 |
+
self.rows = int(matrix.shape[0])
|
| 164 |
+
# A repeated load may be reading replaced files; make every
|
| 165 |
+
# thread reopen on next db() the same way a reload does.
|
| 166 |
+
self.invalidate_connections()
|
| 167 |
+
if self.adapter is not None:
|
| 168 |
+
self.facets = self.adapter.compute_facets(
|
| 169 |
+
self.db(), self.rows)
|
| 170 |
+
self.ready.set()
|
| 171 |
+
except Exception as e: # noqa: BLE001 β isolation is the contract
|
| 172 |
+
self.error = f'{type(e).__name__}: {e}'
|
| 173 |
+
|
| 174 |
+
def reload(self, downloader=None) -> bool:
|
| 175 |
+
"""Synchronize this corpus from the Hub and hot-swap it in place.
|
| 176 |
+
|
| 177 |
+
Unlike `load`, this never skips the downloader call: the whole point is
|
| 178 |
+
to pick up a freshly published index whose files are already on disk.
|
| 179 |
+
The Hub client still uses its normal metadata diffing, so unchanged
|
| 180 |
+
artifacts are not downloaded again.
|
| 181 |
+
|
| 182 |
+
Sequenced so the slow part happens while the old index is still
|
| 183 |
+
serving β download, matrix load and validation all run first, and
|
| 184 |
+
readiness only drops for the couple of seconds it takes to swap
|
| 185 |
+
pointers and recompute facets. That is a short 503 window instead of
|
| 186 |
+
the alternative, which is answering a query against a new matrix and
|
| 187 |
+
a stale meta.sqlite and returning confidently wrong rows.
|
| 188 |
+
|
| 189 |
+
Never raises. On failure the previous matrix, facets and readiness
|
| 190 |
+
are left untouched and `error` records why: a bad publish must not
|
| 191 |
+
take search down. Returns True only when the swap completed.
|
| 192 |
+
"""
|
| 193 |
+
try:
|
| 194 |
+
if downloader is None:
|
| 195 |
+
from huggingface_hub import snapshot_download
|
| 196 |
+
downloader = snapshot_download
|
| 197 |
+
download_index(downloader, self.spec)
|
| 198 |
+
matrix = np.load(
|
| 199 |
+
self.spec.local_dir / MATRIX_NAME).astype(np.float32)
|
| 200 |
+
_validate_matrix(self.spec, matrix)
|
| 201 |
+
check = sqlite3.connect(
|
| 202 |
+
f'file:{self.spec.local_dir / SQLITE_NAME}?mode=ro', uri=True)
|
| 203 |
+
try:
|
| 204 |
+
_validate_sqlite(self.spec, check, matrix_rows=matrix.shape[0])
|
| 205 |
+
finally:
|
| 206 |
+
check.close()
|
| 207 |
+
except Exception as e: # noqa: BLE001 β keep serving the old index
|
| 208 |
+
self.error = f'reload failed: {type(e).__name__}: {e}'
|
| 209 |
+
return False
|
| 210 |
+
|
| 211 |
+
self.ready.clear()
|
| 212 |
+
try:
|
| 213 |
+
self.matrix = matrix
|
| 214 |
+
self.rows = int(matrix.shape[0])
|
| 215 |
+
# Every thread still holds a connection to the replaced file.
|
| 216 |
+
self.invalidate_connections()
|
| 217 |
+
if self.adapter is not None:
|
| 218 |
+
self.facets = self.adapter.compute_facets(self.db(), self.rows)
|
| 219 |
+
self.error = None
|
| 220 |
+
except Exception as e: # noqa: BLE001 β matrix is valid; serve anyway
|
| 221 |
+
self.error = f'reload facets failed: {type(e).__name__}: {e}'
|
| 222 |
+
finally:
|
| 223 |
+
self.ready.set()
|
| 224 |
+
return True
|
| 225 |
+
|
| 226 |
+
|
| 227 |
+
def download_index(downloader, spec: IndexSpec) -> None:
|
| 228 |
+
"""Pull the corpus files from the Hub into spec.local_dir."""
|
| 229 |
+
downloader(
|
| 230 |
+
spec.dataset_id,
|
| 231 |
+
repo_type='dataset',
|
| 232 |
+
local_dir=spec.local_dir,
|
| 233 |
+
allow_patterns=list(spec.files),
|
| 234 |
+
)
|
| 235 |
+
|
| 236 |
+
|
| 237 |
+
class CorpusAdapter(Protocol):
|
| 238 |
+
"""Schema-specific knowledge for one corpus.
|
| 239 |
+
|
| 240 |
+
The core owns download/load/validate/rank; the adapter owns everything
|
| 241 |
+
that differs between models, kernels, and Docker images.
|
| 242 |
+
"""
|
| 243 |
+
spec: IndexSpec
|
| 244 |
+
card_columns: str # SQL select list for result cards
|
| 245 |
+
result_key: str # response key: 'models' | 'kernels' | ...
|
| 246 |
+
|
| 247 |
+
def extract_constraints(self, req) -> str:
|
| 248 |
+
"""Move structured constraints out of the request/query and into
|
| 249 |
+
the request's filter slots (never loosening explicit ones).
|
| 250 |
+
Returns the remaining text to embed; '' skips the embedding."""
|
| 251 |
+
|
| 252 |
+
def build_where(self, req) -> tuple[str, list]:
|
| 253 |
+
"""SQL WHERE clause + args over this corpus's table."""
|
| 254 |
+
|
| 255 |
+
def compute_facets(self, conn: sqlite3.Connection, total: int) -> dict:
|
| 256 |
+
"""Corpus-shaped facet payload served by /api/facets."""
|
| 257 |
+
|
| 258 |
+
def card(self, row: sqlite3.Row, score: float | None = None) -> dict:
|
| 259 |
+
"""Serialize one metadata row into a result card."""
|
| 260 |
+
|
| 261 |
+
def sort_order(self, sort: str) -> str:
|
| 262 |
+
"""ORDER BY expression for structured (non-semantic) sorts."""
|
| 263 |
+
|
| 264 |
+
|
| 265 |
+
class IndexRegistry:
|
| 266 |
+
"""All corpora: registration, isolated loading, per-index status."""
|
| 267 |
+
|
| 268 |
+
def __init__(self, downloader=None) -> None:
|
| 269 |
+
self._indexes: dict[str, LoadedIndex] = {}
|
| 270 |
+
self.downloader = downloader
|
| 271 |
+
|
| 272 |
+
def register(self, adapter) -> LoadedIndex:
|
| 273 |
+
index = LoadedIndex(adapter.spec, adapter)
|
| 274 |
+
self._indexes[adapter.spec.key] = index
|
| 275 |
+
return index
|
| 276 |
+
|
| 277 |
+
def index(self, key: str) -> LoadedIndex:
|
| 278 |
+
return self._indexes[key]
|
| 279 |
+
|
| 280 |
+
def keys(self) -> list[str]:
|
| 281 |
+
return list(self._indexes)
|
| 282 |
+
|
| 283 |
+
def load_all(self) -> None:
|
| 284 |
+
"""Load every corpus inline; failures are isolated per index."""
|
| 285 |
+
for index in self._indexes.values():
|
| 286 |
+
index.load(self.downloader)
|
| 287 |
+
|
| 288 |
+
def start_loading(self) -> None:
|
| 289 |
+
"""Load every corpus on its own background thread (boot path)."""
|
| 290 |
+
for index in self._indexes.values():
|
| 291 |
+
threading.Thread(
|
| 292 |
+
target=index.load, args=(self.downloader,),
|
| 293 |
+
name=f'bsides-load-{index.spec.key}', daemon=True).start()
|
| 294 |
+
|
| 295 |
+
def status(self) -> dict:
|
| 296 |
+
return {key: {'ready': index.ready.is_set(),
|
| 297 |
+
'rows': index.rows,
|
| 298 |
+
'error': index.error}
|
| 299 |
+
for key, index in self._indexes.items()}
|
| 300 |
+
|
| 301 |
+
|
| 302 |
+
def embed_query(index: LoadedIndex, client, q: str, *,
|
| 303 |
+
credential_name: str = 'OPENAI_API_KEY') -> np.ndarray:
|
| 304 |
+
"""Embed one query against this index's model and matrix dimensions."""
|
| 305 |
+
if client is None:
|
| 306 |
+
raise UnavailableError(
|
| 307 |
+
f'semantic search unavailable: no {credential_name}')
|
| 308 |
+
if index.matrix is None:
|
| 309 |
+
raise UnavailableError(
|
| 310 |
+
'semantic search unavailable: index not loaded')
|
| 311 |
+
resp = client.embeddings.create(
|
| 312 |
+
model=index.spec.embed_model,
|
| 313 |
+
dimensions=int(index.matrix.shape[1]),
|
| 314 |
+
input=q[:MAX_QUERY_CHARS],
|
| 315 |
+
)
|
| 316 |
+
v = np.asarray(resp.data[0].embedding, dtype=np.float32)
|
| 317 |
+
return v / max(np.linalg.norm(v), 1e-8)
|
| 318 |
+
|
| 319 |
+
|
| 320 |
+
def semantic_rank(conn, index: LoadedIndex, adapter: CorpusAdapter, *,
|
| 321 |
+
where: str, args: list, sem_q: str, page: int,
|
| 322 |
+
page_size: int, embed_client,
|
| 323 |
+
credential_name: str = 'OPENAI_API_KEY') -> tuple[int, list[dict]]:
|
| 324 |
+
"""The relevance pipeline, corpus-agnostic.
|
| 325 |
+
|
| 326 |
+
Candidate row ids from the filtered set, one query embedding, matrix
|
| 327 |
+
scoring, argpartition top-k, then card serialization via the adapter.
|
| 328 |
+
Returns (total, cards).
|
| 329 |
+
"""
|
| 330 |
+
table = index.spec.table
|
| 331 |
+
rows = np.fromiter(
|
| 332 |
+
(x[0] for x in conn.execute(
|
| 333 |
+
f'SELECT row FROM {table} WHERE {where}', args)),
|
| 334 |
+
dtype=np.int64)
|
| 335 |
+
total = len(rows)
|
| 336 |
+
if total == 0:
|
| 337 |
+
return 0, []
|
| 338 |
+
qv = embed_query(
|
| 339 |
+
index, embed_client, sem_q, credential_name=credential_name)
|
| 340 |
+
mat = index.matrix
|
| 341 |
+
scores = mat[rows] @ qv if total < index.rows // 2 else (mat @ qv)[rows]
|
| 342 |
+
want = min(total, (page + 1) * page_size)
|
| 343 |
+
top = np.argpartition(-scores, want - 1)[:want]
|
| 344 |
+
top = top[np.argsort(-scores[top])][page * page_size:]
|
| 345 |
+
if len(top) == 0:
|
| 346 |
+
return int(total), []
|
| 347 |
+
sel_rows = rows[top]
|
| 348 |
+
sel_scores = scores[top]
|
| 349 |
+
placeholders = ','.join('?' * len(sel_rows))
|
| 350 |
+
by_row = {row['row']: row for row in conn.execute(
|
| 351 |
+
f'SELECT {adapter.card_columns} FROM {table} '
|
| 352 |
+
f'WHERE row IN ({placeholders})', [int(x) for x in sel_rows])}
|
| 353 |
+
cards = [adapter.card(by_row[int(rw)], sc)
|
| 354 |
+
for rw, sc in zip(sel_rows, sel_scores)]
|
| 355 |
+
return int(total), cards
|
| 356 |
+
|
| 357 |
+
|
| 358 |
+
def structured_page(conn, index: LoadedIndex, adapter: CorpusAdapter, *,
|
| 359 |
+
where: str, args: list, sort: str, page: int,
|
| 360 |
+
page_size: int) -> tuple[int, list[dict]]:
|
| 361 |
+
"""Non-semantic path: COUNT + ORDER BY with LIMIT/OFFSET paging."""
|
| 362 |
+
table = index.spec.table
|
| 363 |
+
order = adapter.sort_order(sort)
|
| 364 |
+
total = conn.execute(
|
| 365 |
+
f'SELECT COUNT(*) FROM {table} WHERE {where}', args).fetchone()[0]
|
| 366 |
+
cur = conn.execute(
|
| 367 |
+
f'SELECT {adapter.card_columns} FROM {table} WHERE {where} '
|
| 368 |
+
f'ORDER BY {order} LIMIT ? OFFSET ?',
|
| 369 |
+
args + [page_size, page * page_size])
|
| 370 |
+
cards = [adapter.card(row) for row in cur]
|
| 371 |
+
return int(total), cards
|
| 372 |
+
|
| 373 |
+
|
| 374 |
+
|
| 375 |
+
# ββ models corpus βββββββββββββββοΏ½οΏ½βββββββββββββββββββββββββββββββββββββββββββββ
|
| 376 |
+
# Everything below reproduces the historical single-index model behavior
|
| 377 |
+
# verbatim β request shape, query-constraint parsing, SQL filtering, facet
|
| 378 |
+
# computation and card serialization.
|
| 379 |
+
|
| 380 |
+
MODEL_CARD_COLUMNS = ('row, model_id, author, month, downloads, likes, tags, '
|
| 381 |
+
'base_model, relation, license, params, model_type, '
|
| 382 |
+
'architectures, context_len, num_experts, code_imports, '
|
| 383 |
+
'gated')
|
| 384 |
+
|
| 385 |
+
MODEL_SORT_ORDERS = {
|
| 386 |
+
'newest': 'created_at DESC', 'oldest': 'created_at ASC',
|
| 387 |
+
'downloads': 'downloads DESC', 'likes': 'likes DESC',
|
| 388 |
+
}
|
| 389 |
+
|
| 390 |
+
MODEL_FAMILY_TERMS = {
|
| 391 |
+
'llama': ('llama',),
|
| 392 |
+
'qwen': ('qwen',),
|
| 393 |
+
'gemma': ('gemma',),
|
| 394 |
+
'mistral': ('mistral', 'mixtral'),
|
| 395 |
+
'deepseek': ('deepseek',),
|
| 396 |
+
'phi': ('phi',),
|
| 397 |
+
'granite': ('granite',),
|
| 398 |
+
}
|
| 399 |
+
MODEL_FAMILY_COLUMNS = (
|
| 400 |
+
'model_id', 'model_type', 'tags', 'base_model', 'architectures')
|
| 401 |
+
MODEL_QUANTIZATION_MARKERS = (
|
| 402 |
+
'awq', 'gptq', 'gguf', 'ggml', 'exl2', 'exllama', 'bnb',
|
| 403 |
+
'bitsandbytes', 'hqq', 'quanto', 'aqlm', 'spqr', 'marlin',
|
| 404 |
+
'int4', '4bit', '4-bit', '8bit', '8-bit')
|
| 405 |
+
|
| 406 |
+
METHOD_TAGS = ['sft', 'grpo', 'dpo', 'orpo', 'kto', 'ppo', 'lora', 'merge',
|
| 407 |
+
'unsloth', 'trl', 'axolotl', 'llama-factory', 'autotrain',
|
| 408 |
+
'custom_code']
|
| 409 |
+
|
| 410 |
+
# methods a user might actually type into the query bar (skip hyphenated/internal ids)
|
| 411 |
+
QUERY_METHODS = ['sft', 'grpo', 'dpo', 'orpo', 'kto', 'ppo', 'lora', 'merge',
|
| 412 |
+
'unsloth', 'axolotl', 'autotrain']
|
| 413 |
+
|
| 414 |
+
_MONTHS = {m[:3]: i for i, m in enumerate(
|
| 415 |
+
['jan', 'feb', 'mar', 'apr', 'may', 'jun', 'jul',
|
| 416 |
+
'aug', 'sep', 'oct', 'nov', 'dec'], 1)}
|
| 417 |
+
|
| 418 |
+
# words that carry no meaning once constraints are stripped β drop before embedding
|
| 419 |
+
_STOP = {'a', 'an', 'the', 'from', 'with', 'and', 'or', 'of', 'for', 'in', 'on',
|
| 420 |
+
'to', 'at', 'by', 'that', 'this', 'is', 'are', 'has', 'have', 'than',
|
| 421 |
+
'less', 'more', 'under', 'over', 'below', 'above', 'model', 'models',
|
| 422 |
+
'params', 'param', 'downloads', 'download', 'dl', 'billion', 'b'}
|
| 423 |
+
|
| 424 |
+
ARCH_TAGS = {
|
| 425 |
+
'llama', 'qwen2', 'qwen3', 'gpt2', 'mistral', 'gemma', 'gemma2', 'gemma3_text',
|
| 426 |
+
'stablelm', 'gpt_neox', 'phi', 'phi3', 'mixtral', 'falcon', 'mamba', 'mamba2',
|
| 427 |
+
'jamba', 'rwkv', 'olmo', 'olmo2', 'bloom', 'opt', 'gptj', 'gpt_bigcode',
|
| 428 |
+
'starcoder2', 'deepseek_v2', 'deepseek_v3', 'cohere', 'dbrx', 'exaone',
|
| 429 |
+
'granite', 'internlm2', 'minicpm', 'nemotron', 'persimmon', 'plamo',
|
| 430 |
+
'recurrent_gemma', 'smollm', 'xglm', 'baichuan', 'chatglm', 'yi',
|
| 431 |
+
}
|
| 432 |
+
|
| 433 |
+
|
| 434 |
+
class SearchReq(BaseModel):
|
| 435 |
+
query: str = ''
|
| 436 |
+
filters: dict[str, object] | None = None
|
| 437 |
+
month_from: str | None = None
|
| 438 |
+
month_to: str | None = None
|
| 439 |
+
archs: list[str] = Field(default_factory=list)
|
| 440 |
+
methods: list[str] = Field(default_factory=list)
|
| 441 |
+
relation: str | None = None
|
| 442 |
+
base_model: str | None = None # substring match
|
| 443 |
+
author: str | None = None # substring match
|
| 444 |
+
license: str | None = None
|
| 445 |
+
imports: list[str] = Field(default_factory=list) # code_imports must contain all
|
| 446 |
+
params_min: int | None = None
|
| 447 |
+
params_max: int | None = None
|
| 448 |
+
context_min: int | None = None
|
| 449 |
+
downloads_min: int | None = None
|
| 450 |
+
downloads_max: int | None = None
|
| 451 |
+
moe_only: bool = False
|
| 452 |
+
custom_code_only: bool = False
|
| 453 |
+
config_contains: str | None = None # raw substring over full config json
|
| 454 |
+
exclude_families: list[str] = Field(default_factory=list)
|
| 455 |
+
exclude_quantizations: bool = False
|
| 456 |
+
sort: str = 'relevance' # relevance | newest | oldest | downloads | likes
|
| 457 |
+
page: int = 0
|
| 458 |
+
|
| 459 |
+
|
| 460 |
+
|
| 461 |
+
def parse_query_constraints(q: str) -> tuple[dict, str]:
|
| 462 |
+
"""Pull explicit hard constraints (dates, download/param ranges, methods,
|
| 463 |
+
custom-code) out of a natural-language query so they become SQL filters
|
| 464 |
+
instead of being fed to the embedder as vague semantic vibes.
|
| 465 |
+
|
| 466 |
+
Returns (extracted, remainder) where `extracted` holds SearchReq-style
|
| 467 |
+
overrides and `remainder` is the conceptual text to embed. Only the slots
|
| 468 |
+
the caller leaves empty should be filled β never loosen an explicit facet.
|
| 469 |
+
"""
|
| 470 |
+
ex: dict = {}
|
| 471 |
+
work = q
|
| 472 |
+
|
| 473 |
+
# ββ date: "feb 2025" (single month) else bare "2024" (whole year) ββ
|
| 474 |
+
m = re.search(r'\b(jan|feb|mar|apr|may|jun|jul|aug|sep|oct|nov|dec)'
|
| 475 |
+
r'[a-z]*\.?\s+(20\d\d)\b', work)
|
| 476 |
+
if m:
|
| 477 |
+
ym = f'{m.group(2)}-{_MONTHS[m.group(1)]:02d}'
|
| 478 |
+
ex['month_from'] = ex['month_to'] = ym
|
| 479 |
+
work = work[:m.start()] + ' ' + work[m.end():]
|
| 480 |
+
else:
|
| 481 |
+
# Require a word boundary so ISO dates like 2025-05 don't have the
|
| 482 |
+
# year extracted from under the reader's feet; a full YYYY-MM string
|
| 483 |
+
# means the caller already knows the month and the parser should not
|
| 484 |
+
# widen it to the whole year.
|
| 485 |
+
m = re.search(r'\b(20\d\d)\b', work)
|
| 486 |
+
if m:
|
| 487 |
+
end = m.end()
|
| 488 |
+
if end < len(work) and work[end] == '-':
|
| 489 |
+
pass
|
| 490 |
+
else:
|
| 491 |
+
ex['month_from'] = f'{m.group(1)}-01'
|
| 492 |
+
ex['month_to'] = f'{m.group(1)}-12'
|
| 493 |
+
work = work[:m.start()] + ' ' + work[m.end():]
|
| 494 |
+
|
| 495 |
+
# ββ downloads: "10-50 downloads" range, then < / > comparators (need unit) ββ
|
| 496 |
+
m = re.search(r'\b(\d+)\s*(?:-|to|β|β)\s*(\d+)\s*downloads?\b', work)
|
| 497 |
+
if m:
|
| 498 |
+
ex['downloads_min'] = int(m.group(1))
|
| 499 |
+
ex['downloads_max'] = int(m.group(2))
|
| 500 |
+
work = work[:m.start()] + ' ' + work[m.end():]
|
| 501 |
+
else:
|
| 502 |
+
m = re.search(r'\b(?:less\s+than|fewer\s+than|under|below|<)\s*(\d+)\s*'
|
| 503 |
+
r'(?:downloads?|dl|β)\b', work)
|
| 504 |
+
if m:
|
| 505 |
+
ex['downloads_max'] = int(m.group(1)) - 1 # strict: "less than N" β < N
|
| 506 |
+
work = work[:m.start()] + ' ' + work[m.end():]
|
| 507 |
+
else:
|
| 508 |
+
m = re.search(r'\b(?:more\s+than|over|above|>)\s*(\d+)\s*'
|
| 509 |
+
r'(?:downloads?|dl|β)\b', work)
|
| 510 |
+
if m:
|
| 511 |
+
ex['downloads_min'] = int(m.group(1)) + 1 # strict: "more than N" β > N
|
| 512 |
+
work = work[:m.start()] + ' ' + work[m.end():]
|
| 513 |
+
|
| 514 |
+
# ββ params in B: "under 3b", "<3b params", "over 7b" (needs the B suffix) ββ
|
| 515 |
+
m = re.search(r'\b(?:under|less\s+than|below|<|β€)\s*(\d+(?:\.\d+)?)\s*'
|
| 516 |
+
r'[bB](?:illion|b)?\s*(?:params?|p)?\b', work)
|
| 517 |
+
if m:
|
| 518 |
+
ex['params_max'] = int(float(m.group(1)) * 1e9)
|
| 519 |
+
work = work[:m.start()] + ' ' + work[m.end():]
|
| 520 |
+
else:
|
| 521 |
+
m = re.search(r'\b(?:over|more\s+than|above|>|β₯|at\s+least)\s*(\d+(?:\.\d+)?)\s*'
|
| 522 |
+
r'[bB](?:illion|b)?\s*(?:params?|p)?\b', work)
|
| 523 |
+
if m:
|
| 524 |
+
ex['params_min'] = int(float(m.group(1)) * 1e9)
|
| 525 |
+
work = work[:m.start()] + ' ' + work[m.end():]
|
| 526 |
+
|
| 527 |
+
# ββ custom code: "custom code" / "custom-code" ββ
|
| 528 |
+
if re.search(r'\bcustom[ -]?code\b', work):
|
| 529 |
+
ex['custom_code_only'] = True
|
| 530 |
+
work = re.sub(r'\bcustom[ -]?code\b', ' ', work)
|
| 531 |
+
|
| 532 |
+
# ββ method tags (literal words; left in the embedding β they're meaning) ββ
|
| 533 |
+
ex['methods'] = [mt for mt in QUERY_METHODS
|
| 534 |
+
if re.search(r'\b' + re.escape(mt) + r'\b', work)]
|
| 535 |
+
|
| 536 |
+
remainder = re.sub(r'\s+', ' ', work).strip()
|
| 537 |
+
meaningful = [w for w in remainder.split() if w.lower() not in _STOP and len(w) > 2]
|
| 538 |
+
# if nothing conceptual survives, return an empty remainder so the caller
|
| 539 |
+
# skips the embedding call and just sorts the filtered set
|
| 540 |
+
return ex, (' '.join(meaningful) if meaningful else '')
|
| 541 |
+
|
| 542 |
+
|
| 543 |
+
|
| 544 |
+
class ModelAdapter:
|
| 545 |
+
"""The models corpus: today's single-index behavior, relocated intact."""
|
| 546 |
+
|
| 547 |
+
card_columns = MODEL_CARD_COLUMNS
|
| 548 |
+
result_key = 'models'
|
| 549 |
+
|
| 550 |
+
def __init__(self, spec: IndexSpec) -> None:
|
| 551 |
+
self.spec = spec
|
| 552 |
+
|
| 553 |
+
def extract_constraints(self, r: SearchReq) -> str:
|
| 554 |
+
"""Parse the query, fill only the slots left empty, return the
|
| 555 |
+
remaining semantic text ('' skips the embedding call)."""
|
| 556 |
+
ex, sem_q = parse_query_constraints(r.query.strip())
|
| 557 |
+
if ex.get('month_from') and not r.month_from:
|
| 558 |
+
r.month_from = ex['month_from']
|
| 559 |
+
if ex.get('month_to') and not r.month_to:
|
| 560 |
+
r.month_to = ex['month_to']
|
| 561 |
+
if ex.get('downloads_min') is not None and r.downloads_min is None:
|
| 562 |
+
r.downloads_min = ex['downloads_min']
|
| 563 |
+
if ex.get('downloads_max') is not None and r.downloads_max is None:
|
| 564 |
+
r.downloads_max = ex['downloads_max']
|
| 565 |
+
if ex.get('params_min') is not None and r.params_min is None:
|
| 566 |
+
r.params_min = ex['params_min']
|
| 567 |
+
if ex.get('params_max') is not None and r.params_max is None:
|
| 568 |
+
r.params_max = ex['params_max']
|
| 569 |
+
if ex.get('custom_code_only'):
|
| 570 |
+
r.custom_code_only = True
|
| 571 |
+
for mth in ex.get('methods', []):
|
| 572 |
+
if mth not in r.methods:
|
| 573 |
+
r.methods.append(mth)
|
| 574 |
+
return sem_q
|
| 575 |
+
|
| 576 |
+
def build_where(self, r: SearchReq) -> tuple[str, list]:
|
| 577 |
+
where, args = ['1=1'], []
|
| 578 |
+
if r.month_from:
|
| 579 |
+
where.append('month >= ?'); args.append(r.month_from)
|
| 580 |
+
if r.month_to:
|
| 581 |
+
where.append('month <= ?'); args.append(r.month_to)
|
| 582 |
+
if r.archs:
|
| 583 |
+
ors = []
|
| 584 |
+
for a in r.archs:
|
| 585 |
+
ors.append('(model_type = ? OR tags LIKE ?)')
|
| 586 |
+
args += [a, f'%"{a}"%']
|
| 587 |
+
where.append('(' + ' OR '.join(ors) + ')')
|
| 588 |
+
for m in r.methods:
|
| 589 |
+
where.append('tags LIKE ?'); args.append(f'%"{m}"%')
|
| 590 |
+
if r.relation:
|
| 591 |
+
where.append('relation = ?'); args.append(r.relation)
|
| 592 |
+
if r.base_model:
|
| 593 |
+
where.append('base_model LIKE ?'); args.append(f'%{r.base_model}%')
|
| 594 |
+
if r.author:
|
| 595 |
+
where.append('author LIKE ?'); args.append(f'%{r.author}%')
|
| 596 |
+
if r.license:
|
| 597 |
+
where.append('license = ?'); args.append(r.license)
|
| 598 |
+
for imp in r.imports:
|
| 599 |
+
where.append('code_imports LIKE ?'); args.append(f'%"{imp}"%')
|
| 600 |
+
if r.params_min is not None:
|
| 601 |
+
where.append('params >= ?'); args.append(r.params_min)
|
| 602 |
+
if r.params_max is not None:
|
| 603 |
+
where.append('params <= ?'); args.append(r.params_max)
|
| 604 |
+
if r.context_min is not None:
|
| 605 |
+
where.append('context_len >= ?'); args.append(r.context_min)
|
| 606 |
+
if r.downloads_min is not None:
|
| 607 |
+
where.append('downloads >= ?'); args.append(r.downloads_min)
|
| 608 |
+
if r.downloads_max is not None:
|
| 609 |
+
where.append('downloads <= ?'); args.append(r.downloads_max)
|
| 610 |
+
if r.moe_only:
|
| 611 |
+
where.append('num_experts > 1')
|
| 612 |
+
if r.custom_code_only:
|
| 613 |
+
where.append("(code_imports IS NOT NULL OR tags LIKE '%\"custom_code\"%')")
|
| 614 |
+
if r.config_contains:
|
| 615 |
+
where.append('config LIKE ?'); args.append(f'%{r.config_contains}%')
|
| 616 |
+
for family in r.exclude_families:
|
| 617 |
+
terms = MODEL_FAMILY_TERMS.get(family.lower())
|
| 618 |
+
if not terms:
|
| 619 |
+
continue
|
| 620 |
+
family_args = []
|
| 621 |
+
rendered = []
|
| 622 |
+
for column_name in MODEL_FAMILY_COLUMNS:
|
| 623 |
+
for term in terms:
|
| 624 |
+
rendered.append(
|
| 625 |
+
f"LOWER(COALESCE({column_name}, '')) LIKE ?")
|
| 626 |
+
family_args.append(f'%{term}%')
|
| 627 |
+
where.append('NOT (' + ' OR '.join(rendered) + ')')
|
| 628 |
+
args.extend(family_args)
|
| 629 |
+
if r.exclude_quantizations:
|
| 630 |
+
rendered = []
|
| 631 |
+
quant_args = []
|
| 632 |
+
for column_name in MODEL_FAMILY_COLUMNS:
|
| 633 |
+
for marker in MODEL_QUANTIZATION_MARKERS:
|
| 634 |
+
rendered.append(
|
| 635 |
+
f"LOWER(COALESCE({column_name}, '')) LIKE ?")
|
| 636 |
+
quant_args.append(f'%{marker}%')
|
| 637 |
+
rendered.append(
|
| 638 |
+
"CASE WHEN json_valid(config) "
|
| 639 |
+
"THEN json_extract(config, '$.quantization_config') IS NOT NULL "
|
| 640 |
+
"ELSE 0 END")
|
| 641 |
+
where.append('NOT (' + ' OR '.join(rendered) + ')')
|
| 642 |
+
args.extend(quant_args)
|
| 643 |
+
return ' AND '.join(where), args
|
| 644 |
+
|
| 645 |
+
def sort_order(self, sort: str) -> str:
|
| 646 |
+
return MODEL_SORT_ORDERS.get(sort, 'created_at DESC')
|
| 647 |
+
|
| 648 |
+
def card(self, row: sqlite3.Row, score: float | None = None) -> dict:
|
| 649 |
+
d = dict(row)
|
| 650 |
+
d['tags'] = orjson.loads(d['tags']) if d['tags'] else []
|
| 651 |
+
d['code_imports'] = orjson.loads(d['code_imports']) if d['code_imports'] else []
|
| 652 |
+
d['architectures'] = orjson.loads(d['architectures']) if d['architectures'] else []
|
| 653 |
+
if score is not None:
|
| 654 |
+
d['score'] = round(float(score), 4)
|
| 655 |
+
return d
|
| 656 |
+
|
| 657 |
+
|
| 658 |
+
def compute_facets(self, c: sqlite3.Connection, total: int) -> dict:
|
| 659 |
+
months = [r[0] for r in c.execute(
|
| 660 |
+
'SELECT DISTINCT month FROM models ORDER BY month')]
|
| 661 |
+
licenses = [{'v': r[0], 'n': r[1]} for r in c.execute(
|
| 662 |
+
'SELECT license, COUNT(*) FROM models WHERE license IS NOT NULL '
|
| 663 |
+
'GROUP BY license ORDER BY 2 DESC LIMIT 30')]
|
| 664 |
+
relations = [{'v': r[0], 'n': r[1]} for r in c.execute(
|
| 665 |
+
'SELECT relation, COUNT(*) FROM models WHERE relation IS NOT NULL '
|
| 666 |
+
'GROUP BY relation ORDER BY 2 DESC')]
|
| 667 |
+
# Prolific authors are a real cluster axis: one person's run of regional
|
| 668 |
+
# language finetunes, one lab's architecture probes.
|
| 669 |
+
authors = [{'v': r[0], 'n': r[1]} for r in c.execute(
|
| 670 |
+
'SELECT author, COUNT(*) FROM models WHERE author IS NOT NULL '
|
| 671 |
+
'GROUP BY author ORDER BY 2 DESC LIMIT 40')]
|
| 672 |
+
|
| 673 |
+
enriched = c.execute(
|
| 674 |
+
'SELECT COUNT(*) FROM models WHERE model_type IS NOT NULL').fetchone()[0]
|
| 675 |
+
if enriched:
|
| 676 |
+
archs = [{'v': r[0], 'n': r[1]} for r in c.execute(
|
| 677 |
+
'SELECT model_type, COUNT(*) FROM models '
|
| 678 |
+
'WHERE model_type IS NOT NULL '
|
| 679 |
+
'GROUP BY model_type ORDER BY 2 DESC LIMIT 40')]
|
| 680 |
+
imports = {}
|
| 681 |
+
for (ci,) in c.execute(
|
| 682 |
+
'SELECT code_imports FROM models '
|
| 683 |
+
'WHERE code_imports IS NOT NULL'):
|
| 684 |
+
for mod in orjson.loads(ci):
|
| 685 |
+
imports[mod] = imports.get(mod, 0) + 1
|
| 686 |
+
top_imports = [{'v': k, 'n': v} for k, v in
|
| 687 |
+
sorted(imports.items(), key=lambda x: -x[1])[:40]]
|
| 688 |
+
else:
|
| 689 |
+
# pre-enrichment fallback: architecture tags from the hub tag vocabulary
|
| 690 |
+
arch_counts = {}
|
| 691 |
+
for (tags,) in c.execute('SELECT tags FROM models'):
|
| 692 |
+
for t in orjson.loads(tags):
|
| 693 |
+
if t in ARCH_TAGS:
|
| 694 |
+
arch_counts[t] = arch_counts.get(t, 0) + 1
|
| 695 |
+
archs = [{'v': k, 'n': v} for k, v in
|
| 696 |
+
sorted(arch_counts.items(), key=lambda x: -x[1])[:40]]
|
| 697 |
+
top_imports = []
|
| 698 |
+
|
| 699 |
+
methods = []
|
| 700 |
+
for m in METHOD_TAGS:
|
| 701 |
+
n = c.execute('SELECT COUNT(*) FROM models WHERE tags LIKE ?',
|
| 702 |
+
(f'%"{m}"%',)).fetchone()[0]
|
| 703 |
+
if n:
|
| 704 |
+
methods.append({'v': m, 'n': n})
|
| 705 |
+
|
| 706 |
+
return {
|
| 707 |
+
'total': total, 'months': months, 'archs': archs,
|
| 708 |
+
'methods': methods, 'licenses': licenses, 'relations': relations,
|
| 709 |
+
'imports': top_imports, 'authors': authors,
|
| 710 |
+
'enriched': bool(enriched),
|
| 711 |
+
}
|
| 712 |
+
|
| 713 |
+
|
| 714 |
+
# ββ kernels corpus ββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 715 |
+
|
| 716 |
+
KERNEL_CARD_COLUMNS = (
|
| 717 |
+
'row, repo_id, author, created_at, month, last_modified, downloads, '
|
| 718 |
+
'likes, tags, languages, accelerators, torch_versions, cuda_archs, '
|
| 719 |
+
'cpu_archs, operating_systems, dtypes, intrinsics, kernel_names, '
|
| 720 |
+
'torch_ops, build_variants, variant_count, has_build_dir, source_files'
|
| 721 |
+
)
|
| 722 |
+
|
| 723 |
+
KERNEL_JSON_COLUMNS = (
|
| 724 |
+
'tags', 'languages', 'accelerators', 'torch_versions', 'cuda_archs',
|
| 725 |
+
'cpu_archs', 'operating_systems', 'dtypes', 'intrinsics', 'kernel_names',
|
| 726 |
+
'torch_ops', 'build_variants',
|
| 727 |
+
)
|
| 728 |
+
|
| 729 |
+
KERNEL_ARRAY_FILTERS = (
|
| 730 |
+
'languages', 'accelerators', 'cuda_archs', 'dtypes', 'intrinsics',
|
| 731 |
+
'torch_ops', 'torch_versions',
|
| 732 |
+
)
|
| 733 |
+
|
| 734 |
+
KERNEL_SORT_ORDERS = {
|
| 735 |
+
'newest': 'created_at DESC', 'oldest': 'created_at ASC',
|
| 736 |
+
'downloads': 'downloads DESC', 'likes': 'likes DESC',
|
| 737 |
+
}
|
| 738 |
+
|
| 739 |
+
|
| 740 |
+
class KernelAdapter:
|
| 741 |
+
"""Search behavior for the published kernel-repository index."""
|
| 742 |
+
|
| 743 |
+
card_columns = KERNEL_CARD_COLUMNS
|
| 744 |
+
result_key = 'kernels'
|
| 745 |
+
|
| 746 |
+
def __init__(self, spec: IndexSpec) -> None:
|
| 747 |
+
self.spec = spec
|
| 748 |
+
|
| 749 |
+
def extract_constraints(self, r: SearchReq) -> str:
|
| 750 |
+
# Kernel-specific structure is explicit in `filters`; query prose stays
|
| 751 |
+
# intact for semantic ranking instead of entering the model parser.
|
| 752 |
+
return r.query.strip()
|
| 753 |
+
|
| 754 |
+
def build_where(self, r: SearchReq) -> tuple[str, list]:
|
| 755 |
+
filters = r.filters or {}
|
| 756 |
+
where, args = ['1=1'], []
|
| 757 |
+
|
| 758 |
+
author = filters.get('author')
|
| 759 |
+
if isinstance(author, str) and author:
|
| 760 |
+
where.append('author LIKE ?')
|
| 761 |
+
args.append(f'%{author}%')
|
| 762 |
+
|
| 763 |
+
# month is stored as 'YYYY-MM' text; reject anything that is not
|
| 764 |
+
# zero-padded so a fat-fingered '2024-1' filter can't silently match
|
| 765 |
+
# nothing instead of erroring where the caller can see it.
|
| 766 |
+
for key in ('month_from', 'month_to'):
|
| 767 |
+
value = filters.get(key)
|
| 768 |
+
if value is None:
|
| 769 |
+
continue
|
| 770 |
+
if (isinstance(value, str)
|
| 771 |
+
and re.fullmatch(r'\d{4}-(0[1-9]|1[0-2])', value)):
|
| 772 |
+
where.append(f"month {('>=', '<=')[key == 'month_to']} ?")
|
| 773 |
+
args.append(value)
|
| 774 |
+
else:
|
| 775 |
+
raise UnavailableError(
|
| 776 |
+
f'{key} filter must be a zero-padded YYYY-MM string')
|
| 777 |
+
|
| 778 |
+
# JSON has no int type, so a UI that ships '50' as a string must not
|
| 779 |
+
# end up comparing a TEXT arg against an INTEGER column (which in
|
| 780 |
+
# SQLite never matches and returns an empty page with no error).
|
| 781 |
+
for key, column in (('downloads_min', 'downloads'),
|
| 782 |
+
('downloads_max', 'downloads'),
|
| 783 |
+
('variant_count_min', 'variant_count')):
|
| 784 |
+
value = filters.get(key)
|
| 785 |
+
if value is None:
|
| 786 |
+
continue
|
| 787 |
+
operator = '<=' if key.endswith('_max') else '>='
|
| 788 |
+
try:
|
| 789 |
+
coerced = int(value)
|
| 790 |
+
except (TypeError, ValueError) as exc:
|
| 791 |
+
raise UnavailableError(
|
| 792 |
+
f'{key} filter must be an integer') from exc
|
| 793 |
+
where.append(f'{column} {operator} ?')
|
| 794 |
+
args.append(coerced)
|
| 795 |
+
|
| 796 |
+
for column in KERNEL_ARRAY_FILTERS:
|
| 797 |
+
selected = filters.get(column)
|
| 798 |
+
if not isinstance(selected, (list, tuple)):
|
| 799 |
+
continue
|
| 800 |
+
values = [value for value in selected
|
| 801 |
+
if isinstance(value, str) and value]
|
| 802 |
+
if not values:
|
| 803 |
+
continue
|
| 804 |
+
placeholders = ','.join('?' * len(values))
|
| 805 |
+
where.append(
|
| 806 |
+
f'EXISTS (SELECT 1 FROM json_each(kernels.{column}) AS item '
|
| 807 |
+
f'WHERE item.value IN ({placeholders}))')
|
| 808 |
+
args.extend(values)
|
| 809 |
+
|
| 810 |
+
has_build_dir = filters.get('has_build_dir')
|
| 811 |
+
if isinstance(has_build_dir, bool):
|
| 812 |
+
where.append('has_build_dir = ?')
|
| 813 |
+
args.append(int(has_build_dir))
|
| 814 |
+
|
| 815 |
+
return ' AND '.join(where), args
|
| 816 |
+
|
| 817 |
+
def sort_order(self, sort: str) -> str:
|
| 818 |
+
return KERNEL_SORT_ORDERS.get(sort, 'created_at DESC')
|
| 819 |
+
|
| 820 |
+
def card(self, row: sqlite3.Row, score: float | None = None) -> dict:
|
| 821 |
+
card = dict(row)
|
| 822 |
+
for column in KERNEL_JSON_COLUMNS:
|
| 823 |
+
card[column] = orjson.loads(card[column]) if card[column] else []
|
| 824 |
+
card['has_build_dir'] = bool(card['has_build_dir'])
|
| 825 |
+
if score is not None:
|
| 826 |
+
card['score'] = round(float(score), 4)
|
| 827 |
+
return card
|
| 828 |
+
|
| 829 |
+
def compute_facets(self, c: sqlite3.Connection, total: int) -> dict:
|
| 830 |
+
months = [row[0] for row in c.execute(
|
| 831 |
+
'SELECT DISTINCT month FROM kernels WHERE month IS NOT NULL '
|
| 832 |
+
'ORDER BY month')]
|
| 833 |
+
authors = [{'v': row[0], 'n': row[1]} for row in c.execute(
|
| 834 |
+
'SELECT author, COUNT(*) FROM kernels WHERE author IS NOT NULL '
|
| 835 |
+
'GROUP BY author ORDER BY 2 DESC, 1 ASC LIMIT 40')]
|
| 836 |
+
|
| 837 |
+
facets = {'total': total, 'months': months, 'authors': authors}
|
| 838 |
+
for column in KERNEL_ARRAY_FILTERS:
|
| 839 |
+
counts: dict[str, int] = {}
|
| 840 |
+
for (payload,) in c.execute(
|
| 841 |
+
f'SELECT {column} FROM kernels WHERE {column} IS NOT NULL'):
|
| 842 |
+
for value in orjson.loads(payload):
|
| 843 |
+
counts[value] = counts.get(value, 0) + 1
|
| 844 |
+
facets[column] = [
|
| 845 |
+
{'v': value, 'n': count}
|
| 846 |
+
for value, count in sorted(
|
| 847 |
+
counts.items(), key=lambda item: (-item[1], item[0]))
|
| 848 |
+
]
|
| 849 |
+
|
| 850 |
+
facets['has_build_dir'] = [
|
| 851 |
+
{'v': bool(value), 'n': count}
|
| 852 |
+
for value, count in c.execute(
|
| 853 |
+
'SELECT has_build_dir, COUNT(*) FROM kernels '
|
| 854 |
+
'GROUP BY has_build_dir ORDER BY has_build_dir DESC')
|
| 855 |
+
]
|
| 856 |
+
return facets
|
| 857 |
+
|
| 858 |
+
|
| 859 |
+
# ββ containers corpus ββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 860 |
+
|
| 861 |
+
CONTAINER_CARD_COLUMNS = (
|
| 862 |
+
'row, repo_id, repo_type, artifact_path, artifact_kind, author, sha, '
|
| 863 |
+
'created_at, last_modified, downloads, likes, tags, description, '
|
| 864 |
+
'artifact_files, qualification_reasons, base_images, accelerators, '
|
| 865 |
+
'cuda_versions, rocm_versions, python_versions, node_versions, '
|
| 866 |
+
'operating_systems, ports, services, package_managers, frameworks, '
|
| 867 |
+
'entrypoints, commands, has_compose, has_devcontainer, has_multistage, '
|
| 868 |
+
'source_files, source_bytes'
|
| 869 |
+
)
|
| 870 |
+
|
| 871 |
+
CONTAINER_JSON_COLUMNS = (
|
| 872 |
+
'tags', 'artifact_files', 'qualification_reasons', 'base_images',
|
| 873 |
+
'accelerators', 'cuda_versions', 'rocm_versions', 'python_versions',
|
| 874 |
+
'node_versions', 'operating_systems', 'ports', 'services',
|
| 875 |
+
'package_managers', 'frameworks', 'entrypoints', 'commands',
|
| 876 |
+
)
|
| 877 |
+
|
| 878 |
+
CONTAINER_SCALAR_FILTERS = {
|
| 879 |
+
'repo_types': 'repo_type',
|
| 880 |
+
'artifact_kinds': 'artifact_kind',
|
| 881 |
+
}
|
| 882 |
+
|
| 883 |
+
CONTAINER_ARRAY_FILTERS = (
|
| 884 |
+
'base_images', 'accelerators', 'cuda_versions', 'rocm_versions',
|
| 885 |
+
'python_versions', 'node_versions', 'operating_systems', 'ports',
|
| 886 |
+
'services', 'package_managers', 'frameworks',
|
| 887 |
+
)
|
| 888 |
+
|
| 889 |
+
CONTAINER_BOOL_FILTERS = {
|
| 890 |
+
'has_compose': 'has_compose',
|
| 891 |
+
'has_devcontainer': 'has_devcontainer',
|
| 892 |
+
'has_multistage': 'has_multistage',
|
| 893 |
+
}
|
| 894 |
+
|
| 895 |
+
CONTAINER_SORT_ORDERS = {
|
| 896 |
+
'newest': 'created_at DESC', 'oldest': 'created_at ASC',
|
| 897 |
+
'downloads': 'downloads DESC', 'likes': 'likes DESC',
|
| 898 |
+
}
|
| 899 |
+
|
| 900 |
+
|
| 901 |
+
class ContainerAdapter:
|
| 902 |
+
"""Search behavior for the published container-artifact index."""
|
| 903 |
+
|
| 904 |
+
card_columns = CONTAINER_CARD_COLUMNS
|
| 905 |
+
result_key = 'containers'
|
| 906 |
+
|
| 907 |
+
def __init__(self, spec: IndexSpec) -> None:
|
| 908 |
+
self.spec = spec
|
| 909 |
+
|
| 910 |
+
def extract_constraints(self, r: SearchReq) -> str:
|
| 911 |
+
return r.query.strip()
|
| 912 |
+
|
| 913 |
+
def build_where(self, r: SearchReq) -> tuple[str, list]:
|
| 914 |
+
filters = r.filters or {}
|
| 915 |
+
where, args = ['1=1'], []
|
| 916 |
+
|
| 917 |
+
for key, column in CONTAINER_SCALAR_FILTERS.items():
|
| 918 |
+
selected = filters.get(key)
|
| 919 |
+
if not isinstance(selected, (list, tuple)):
|
| 920 |
+
continue
|
| 921 |
+
values = [value for value in selected
|
| 922 |
+
if isinstance(value, str) and value]
|
| 923 |
+
if not values:
|
| 924 |
+
continue
|
| 925 |
+
placeholders = ','.join('?' * len(values))
|
| 926 |
+
where.append(f'{column} IN ({placeholders})')
|
| 927 |
+
args.extend(values)
|
| 928 |
+
|
| 929 |
+
for column in CONTAINER_ARRAY_FILTERS:
|
| 930 |
+
selected = filters.get(column)
|
| 931 |
+
if not isinstance(selected, (list, tuple)):
|
| 932 |
+
continue
|
| 933 |
+
values = [value for value in selected
|
| 934 |
+
if isinstance(value, (str, int)) and str(value) != '']
|
| 935 |
+
if not values:
|
| 936 |
+
continue
|
| 937 |
+
placeholders = ','.join('?' * len(values))
|
| 938 |
+
# ports are published as JSON integers; comparing the raw value
|
| 939 |
+
# against a TEXT arg via json_each never matches. Normalize both
|
| 940 |
+
# sides to text so '8888' and 8888 hit the same row.
|
| 941 |
+
compare = (f"CAST(item.value AS TEXT) IN ({placeholders})"
|
| 942 |
+
if column == 'ports' else
|
| 943 |
+
f'item.value IN ({placeholders})')
|
| 944 |
+
where.append(
|
| 945 |
+
f'EXISTS (SELECT 1 FROM json_each(containers.{column}) AS item '
|
| 946 |
+
f'WHERE {compare})')
|
| 947 |
+
args.extend([str(v) for v in values] if column == 'ports'
|
| 948 |
+
else values)
|
| 949 |
+
|
| 950 |
+
for key, column in CONTAINER_BOOL_FILTERS.items():
|
| 951 |
+
value = filters.get(key)
|
| 952 |
+
if isinstance(value, bool):
|
| 953 |
+
where.append(f'{column} = ?')
|
| 954 |
+
args.append(int(value))
|
| 955 |
+
|
| 956 |
+
return ' AND '.join(where), args
|
| 957 |
+
|
| 958 |
+
def sort_order(self, sort: str) -> str:
|
| 959 |
+
return CONTAINER_SORT_ORDERS.get(sort, 'created_at DESC')
|
| 960 |
+
|
| 961 |
+
def card(self, row: sqlite3.Row, score: float | None = None) -> dict:
|
| 962 |
+
card = dict(row)
|
| 963 |
+
for column in CONTAINER_JSON_COLUMNS:
|
| 964 |
+
card[column] = orjson.loads(card[column]) if card[column] else []
|
| 965 |
+
for column in CONTAINER_BOOL_FILTERS.values():
|
| 966 |
+
card[column] = bool(card[column])
|
| 967 |
+
if score is not None:
|
| 968 |
+
card['score'] = round(float(score), 4)
|
| 969 |
+
return card
|
| 970 |
+
|
| 971 |
+
def compute_facets(self, c: sqlite3.Connection, total: int) -> dict:
|
| 972 |
+
facets = {'total': total}
|
| 973 |
+
|
| 974 |
+
for key, column in CONTAINER_SCALAR_FILTERS.items():
|
| 975 |
+
facets[key] = [
|
| 976 |
+
{'v': value, 'n': count}
|
| 977 |
+
for value, count in c.execute(
|
| 978 |
+
f'SELECT {column}, COUNT(*) FROM containers '
|
| 979 |
+
f'WHERE {column} IS NOT NULL '
|
| 980 |
+
f'GROUP BY {column} ORDER BY 2 DESC, 1 ASC')
|
| 981 |
+
]
|
| 982 |
+
|
| 983 |
+
for column in CONTAINER_ARRAY_FILTERS:
|
| 984 |
+
counts: dict[str, int] = {}
|
| 985 |
+
for (payload,) in c.execute(
|
| 986 |
+
f'SELECT {column} FROM containers WHERE {column} IS NOT NULL'):
|
| 987 |
+
for value in orjson.loads(payload):
|
| 988 |
+
counts[value] = counts.get(value, 0) + 1
|
| 989 |
+
facets[column] = [
|
| 990 |
+
{'v': value, 'n': count}
|
| 991 |
+
for value, count in sorted(
|
| 992 |
+
counts.items(), key=lambda item: (-item[1], item[0]))
|
| 993 |
+
]
|
| 994 |
+
|
| 995 |
+
for key, column in CONTAINER_BOOL_FILTERS.items():
|
| 996 |
+
facets[key] = [
|
| 997 |
+
{'v': bool(value), 'n': count}
|
| 998 |
+
for value, count in c.execute(
|
| 999 |
+
f'SELECT {column}, COUNT(*) FROM containers '
|
| 1000 |
+
f'GROUP BY {column} ORDER BY {column} DESC')
|
| 1001 |
+
]
|
| 1002 |
+
|
| 1003 |
+
return facets
|
requirements-dev.txt
ADDED
|
@@ -0,0 +1,5 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Local tooling only. Deliberately separate from requirements.txt so the Space
|
| 2 |
+
# image does not carry build/export dependencies it never runs.
|
| 3 |
+
-r requirements.txt
|
| 4 |
+
pyarrow==18.1.0
|
| 5 |
+
pytest==9.1.1
|
requirements.txt
ADDED
|
@@ -0,0 +1,8 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
fastapi==0.141.1
|
| 2 |
+
uvicorn[standard]==0.40.0
|
| 3 |
+
numpy==2.4.2
|
| 4 |
+
orjson==3.11.6
|
| 5 |
+
openai==2.16.0
|
| 6 |
+
huggingface_hub==1.26.0
|
| 7 |
+
# not imported directly; pinned because openai and huggingface_hub both need it
|
| 8 |
+
httpx==0.28.1
|
staff_picks.json
ADDED
|
@@ -0,0 +1,22 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
[
|
| 2 |
+
{
|
| 3 |
+
"model_id": "hoanghai2110/HyperMambaLM-300M",
|
| 4 |
+
"note": "A Mamba state-space backbone with MAML-style meta-learning bolted on β built to few-shot adapt. Custom-code-only, ~300M, 11 downloads; the kind of weird SSM hybrid that never makes the charts.",
|
| 5 |
+
"picked_by": "juiceb0xc0de"
|
| 6 |
+
},
|
| 7 |
+
{
|
| 8 |
+
"model_id": "GSAI-ML/ReFusion",
|
| 9 |
+
"note": "A diffusion language model built on Qwen3-8B out of Renmin's Gaoling School β it denoises toward text instead of predicting the next token. Custom-code, 8B, and it's a mind fuck to get started.",
|
| 10 |
+
"picked_by": "juiceb0xc0de"
|
| 11 |
+
},
|
| 12 |
+
{
|
| 13 |
+
"model_id": "emozilla/LLongMA-2-7b-storysummarizer",
|
| 14 |
+
"note": "An 8K-context Llama fine-tuned on BookSum to summarize stories β a forgotten entry in the LLongMA long-context line. Custom modeling code, 5 downloads, a quiet chapter from the great context-length land grab.",
|
| 15 |
+
"picked_by": "juiceb0xc0de"
|
| 16 |
+
},
|
| 17 |
+
{
|
| 18 |
+
"model_id": "SongTonyLi/OpenELM-270M-CPT-D_chosen-HuggingFaceH4-ultrafeedback_binarized-Xlarge",
|
| 19 |
+
"note": "A 270M OpenELM run through DPO on UltraFeedback β the whole preference-tuning pipeline shrunk to pocket size. Custom-code, 2 downloads, and a repo name that won't fit on the label.",
|
| 20 |
+
"picked_by": "juiceb0xc0de"
|
| 21 |
+
}
|
| 22 |
+
]
|
static/index.html
ADDED
|
@@ -0,0 +1,1217 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
<!DOCTYPE html>
|
| 2 |
+
<html lang="en">
|
| 3 |
+
<head>
|
| 4 |
+
<meta charset="utf-8">
|
| 5 |
+
<meta name="viewport" content="width=device-width, initial-scale=1">
|
| 6 |
+
<title>B-Sides β deep search for indie & vintage LLMs</title>
|
| 7 |
+
<meta name="description" content="Search 228,000+ indie and vintage text-generation models on Hugging Face by meaning, time period, architecture, training method, lineage, config internals like flash_attn β the deep cuts HF search can't reach.">
|
| 8 |
+
<link rel="preconnect" href="https://api.fontshare.com">
|
| 9 |
+
<link href="https://api.fontshare.com/v2/css?f[]=gambarino@400&display=swap" rel="stylesheet">
|
| 10 |
+
<link rel="preconnect" href="https://fonts.googleapis.com">
|
| 11 |
+
<link rel="preconnect" href="https://fonts.gstatic.com" crossorigin>
|
| 12 |
+
<link href="https://fonts.googleapis.com/css2?family=IBM+Plex+Mono:ital,wght@0,400;0,500;0,600;1,400&display=swap" rel="stylesheet">
|
| 13 |
+
<script src="https://cdn.jsdelivr.net/npm/gsap@3.12.5/dist/gsap.min.js"></script>
|
| 14 |
+
<script src="https://cdn.jsdelivr.net/npm/vanilla-tilt@1.8.1/dist/vanilla-tilt.min.js"></script>
|
| 15 |
+
<style>
|
| 16 |
+
:root {
|
| 17 |
+
/* tarot: cream parchment, sepia ink, flat pastel fills */
|
| 18 |
+
--paper: #ece2c8;
|
| 19 |
+
--card: #f6efdc;
|
| 20 |
+
--card2: #f0e7cf;
|
| 21 |
+
--ink: #3a2f22;
|
| 22 |
+
--ink-soft: #675941;
|
| 23 |
+
--ink-faint: #93835f;
|
| 24 |
+
--line: #d5c7a3;
|
| 25 |
+
--frame: #8d7a55;
|
| 26 |
+
--rose: #e9bcc4; --rose-deep: #b25a6e;
|
| 27 |
+
--sage: #c8dcba; --sage-deep: #5f8248;
|
| 28 |
+
--powder: #c2dbe6; --powder-deep: #4f7f96;
|
| 29 |
+
--marigold: #eed093; --marigold-deep: #a1782c;
|
| 30 |
+
--terra: #e7ab95; --terra-deep: #ad4f36;
|
| 31 |
+
--lilac: #d7c7e7; --lilac-deep: #7d5fa0;
|
| 32 |
+
--spring: cubic-bezier(.34,1.56,.64,1);
|
| 33 |
+
--ease: cubic-bezier(.22,.61,.36,1);
|
| 34 |
+
--serif: 'Gambarino', 'Georgia', serif;
|
| 35 |
+
--mono: 'IBM Plex Mono', monospace;
|
| 36 |
+
}
|
| 37 |
+
* { box-sizing: border-box; margin: 0; }
|
| 38 |
+
html { scroll-behavior: smooth; }
|
| 39 |
+
body { background: var(--paper); color: var(--ink); font-family: var(--mono); font-size: 14px; line-height: 1.6; min-height: 100vh; overflow-x: hidden; }
|
| 40 |
+
body::after { /* paper grain */
|
| 41 |
+
content: ''; position: fixed; inset: 0; pointer-events: none; z-index: 1;
|
| 42 |
+
opacity: .05; mix-blend-mode: multiply;
|
| 43 |
+
background-image: url("data:image/svg+xml,%3Csvg xmlns='http://www.w3.org/2000/svg' width='180' height='180'%3E%3Cfilter id='n'%3E%3CfeTurbulence type='fractalNoise' baseFrequency='0.85' numOctaves='3' stitchTiles='stitch'/%3E%3C/filter%3E%3Crect width='180' height='180' filter='url(%23n)'/%3E%3C/svg%3E");
|
| 44 |
+
}
|
| 45 |
+
a { color: var(--terra-deep); text-decoration: none; }
|
| 46 |
+
a:hover { text-decoration: underline; }
|
| 47 |
+
::selection { background: var(--marigold); color: var(--ink); }
|
| 48 |
+
::placeholder { color: var(--ink-faint); }
|
| 49 |
+
input, select, button { font-family: inherit; font-size: inherit; color: inherit; }
|
| 50 |
+
|
| 51 |
+
.wrap { position: relative; z-index: 2; max-width: 1340px; margin: 0 auto; padding: 0 26px; }
|
| 52 |
+
|
| 53 |
+
/* ββ header ββ */
|
| 54 |
+
header { display: flex; align-items: center; justify-content: space-between; padding: 16px 0; border-bottom: 1px solid var(--line); }
|
| 55 |
+
.logo { display: flex; align-items: center; gap: 12px; }
|
| 56 |
+
.logo-disc { width: 32px; height: 32px; border-radius: 50%; position: relative; flex: none;
|
| 57 |
+
background: var(--card); border: 1.5px solid var(--ink); }
|
| 58 |
+
.logo-disc::before { content: ''; position: absolute; inset: 4px; border-radius: 50%; border: 1px solid var(--frame); }
|
| 59 |
+
.logo-disc::after { content: ''; position: absolute; inset: 10px; border-radius: 50%; background: var(--terra); border: 1.5px solid var(--ink); }
|
| 60 |
+
.logo h1 { font-family: var(--serif); font-weight: 400; font-size: 25px; letter-spacing: .5px; }
|
| 61 |
+
.tagline { font-size: 10px; letter-spacing: 2.5px; text-transform: uppercase; color: var(--ink-faint); }
|
| 62 |
+
.head-right { font-size: 11px; letter-spacing: 1px; color: var(--ink-soft); display: flex; gap: 14px; align-items: center; }
|
| 63 |
+
.head-right .dot { color: var(--line); }
|
| 64 |
+
|
| 65 |
+
/* ββ hero: search front and center ββ */
|
| 66 |
+
.hero { display: flex; flex-direction: column; justify-content: center; align-items: center; text-align: center; padding: 34px 0 40px; }
|
| 67 |
+
.hero h2 { font-family: var(--serif); font-weight: 400; font-size: clamp(26px, 3.4vw, 42px); line-height: 1.12; letter-spacing: .2px; max-width: 720px; }
|
| 68 |
+
.hero h2 em { font-style: italic; color: var(--terra-deep); }
|
| 69 |
+
.hero .rule { width: 130px; height: 10px; margin: 14px auto 22px; display: flex; gap: 5px; justify-content: center; }
|
| 70 |
+
.hero .rule i { flex: 1; border-radius: 3px; }
|
| 71 |
+
.searchbar { width: min(720px, 100%); display: flex; align-items: center; gap: 12px;
|
| 72 |
+
background: var(--card); border: 1.5px solid var(--ink); border-radius: 12px;
|
| 73 |
+
padding: 15px 18px; position: relative;
|
| 74 |
+
box-shadow: 4px 4px 0 rgba(58,47,34,.16);
|
| 75 |
+
transition: box-shadow .22s var(--ease), transform .22s var(--ease); }
|
| 76 |
+
.searchbar:focus-within { box-shadow: 6px 7px 0 rgba(58,47,34,.20); transform: translate(-1px,-1px); }
|
| 77 |
+
.searchbar .glyph { color: var(--terra-deep); font-size: 15px; }
|
| 78 |
+
.searchbar input { flex: 1; background: transparent; border: none; outline: none; font-size: 15px; color: var(--ink); min-width: 40px; }
|
| 79 |
+
.corpus-select { flex: none; background: var(--card2); border: 1px solid var(--frame); border-radius: 8px;
|
| 80 |
+
padding: 7px 28px 7px 10px; outline: none; font-size: 11px; font-weight: 600;
|
| 81 |
+
letter-spacing: 1.5px; text-transform: uppercase; cursor: pointer; }
|
| 82 |
+
.corpus-select:focus { border-color: var(--ink); }
|
| 83 |
+
@media (max-width: 620px) {
|
| 84 |
+
.searchbar { gap: 8px; padding: 12px; flex-wrap: wrap; }
|
| 85 |
+
.searchbar .glyph { display: none; }
|
| 86 |
+
.corpus-select { order: 1; }
|
| 87 |
+
.searchbar input { order: 2; width: calc(100% - 118px); }
|
| 88 |
+
.btn-dig { order: 3; margin-left: auto; }
|
| 89 |
+
}
|
| 90 |
+
.btn-dig { border: 1.5px solid var(--ink); cursor: pointer; white-space: nowrap;
|
| 91 |
+
font-size: 11px; font-weight: 600; letter-spacing: 2px; text-transform: uppercase;
|
| 92 |
+
color: var(--ink); background: var(--marigold); border-radius: 9px; padding: 9px 18px;
|
| 93 |
+
box-shadow: 3px 3px 0 var(--ink);
|
| 94 |
+
transition: transform .15s var(--spring), box-shadow .15s var(--spring); }
|
| 95 |
+
.btn-dig:hover { transform: translate(-1px,-2px); box-shadow: 4px 5px 0 var(--ink); }
|
| 96 |
+
.btn-dig:active { transform: translate(3px,3px); box-shadow: 0 0 0 var(--ink); }
|
| 97 |
+
.status-line { margin-top: 14px; font-size: 12.5px; color: var(--ink-soft); min-height: 20px; }
|
| 98 |
+
.status-line b { color: var(--terra-deep); font-weight: 600; }
|
| 99 |
+
.hero .promise { margin-top: 6px; font-size: 11.5px; color: var(--ink-faint); letter-spacing: .4px; max-width: 640px; }
|
| 100 |
+
.hero .promise b { color: var(--ink-soft); font-weight: 500; }
|
| 101 |
+
.hero .manifesto { margin-top: 18px; max-width: 620px; font-family: var(--serif); font-size: 14px; line-height: 1.62; color: var(--ink-soft); }
|
| 102 |
+
|
| 103 |
+
/* ββ layout ββ */
|
| 104 |
+
.main { display: grid; grid-template-columns: 262px 1fr; gap: 30px; padding: 22px 0 80px; }
|
| 105 |
+
@media (max-width: 900px) { .main { grid-template-columns: 1fr; } }
|
| 106 |
+
|
| 107 |
+
/* rail */
|
| 108 |
+
.rail { position: sticky; top: 14px; align-self: start; display: flex; flex-direction: column; gap: 14px; max-height: calc(100vh - 28px); overflow-y: auto; scrollbar-width: thin; padding: 4px 8px 24px 4px; }
|
| 109 |
+
@media (max-width: 900px) { .rail { position: static; max-height: none; } }
|
| 110 |
+
.filter-stack { display: flex; flex-direction: column; gap: 14px; }
|
| 111 |
+
.knob-group { border: 1.5px solid var(--ink); border-radius: 11px; background: var(--card); padding: 13px 14px; box-shadow: 3px 3px 0 rgba(58,47,34,.12); }
|
| 112 |
+
.knob-group h4 { font-size: 10px; letter-spacing: 2.5px; text-transform: uppercase; margin-bottom: 10px; font-weight: 600; color: var(--ink); display: flex; align-items: center; gap: 8px; }
|
| 113 |
+
.knob-group h4 .swatch { width: 22px; height: 8px; border-radius: 3px; border: 1px solid var(--ink); flex: none; }
|
| 114 |
+
.knob-group h4 .soon { margin-left: auto; color: var(--ink-faint); border: 1px dashed var(--frame); border-radius: 6px; padding: 1px 7px; letter-spacing: 1px; font-size: 9px; text-transform: none; }
|
| 115 |
+
.chips { display: flex; flex-wrap: wrap; gap: 6px; }
|
| 116 |
+
.chip { cursor: pointer; user-select: none; font-size: 11.5px; padding: 3.5px 10px; border-radius: 999px;
|
| 117 |
+
border: 1px solid var(--frame); color: var(--ink-soft); background: transparent;
|
| 118 |
+
transition: transform .16s var(--spring), background .16s, color .16s, border-color .16s; }
|
| 119 |
+
.chip:hover { border-color: var(--ink); color: var(--ink); transform: translateY(-1px); }
|
| 120 |
+
.chip .n { opacity: .55; font-size: 10px; margin-left: 4px; }
|
| 121 |
+
.chip.more-toggle { border-style: dashed; }
|
| 122 |
+
.chip.on { border-color: var(--ink); color: var(--ink); font-weight: 600; box-shadow: 2px 2px 0 rgba(58,47,34,.25); }
|
| 123 |
+
.kg-arch .chip.on { background: var(--marigold); }
|
| 124 |
+
.kg-method .chip.on { background: var(--sage); }
|
| 125 |
+
.kg-deep .chip.on { background: var(--lilac); }
|
| 126 |
+
.kg-kernel .chip.on { background: var(--chip-fill, var(--powder)); }
|
| 127 |
+
.rail select, .rail input[type=text], .rail input[type=number] {
|
| 128 |
+
width: 100%; background: #fbf6e8; border: 1px solid var(--frame); border-radius: 8px;
|
| 129 |
+
padding: 7px 10px; outline: none; font-size: 12.5px; color: var(--ink); transition: border-color .2s; }
|
| 130 |
+
.rail select:focus, .rail input:focus { border-color: var(--ink); }
|
| 131 |
+
.range-row { display: flex; gap: 8px; align-items: center; }
|
| 132 |
+
.range-row span { color: var(--ink-faint); font-size: 11px; }
|
| 133 |
+
.toggle-row { display: flex; align-items: center; justify-content: space-between; margin-top: 9px; font-size: 12.5px; color: var(--ink-soft); cursor: pointer; user-select: none; }
|
| 134 |
+
.switch { width: 34px; height: 19px; border-radius: 999px; background: #fbf6e8; border: 1px solid var(--frame); position: relative; transition: background .2s; flex: none; }
|
| 135 |
+
.switch::after { content: ''; position: absolute; top: 2px; left: 2px; width: 13px; height: 13px; border-radius: 50%; background: var(--ink-faint); transition: all .2s var(--spring); }
|
| 136 |
+
.toggle-row.on .switch { background: var(--lilac); border-color: var(--ink); }
|
| 137 |
+
.toggle-row.on .switch::after { left: 17px; background: var(--ink); }
|
| 138 |
+
.sub-label { font-size: 10px; letter-spacing: 1.6px; text-transform: uppercase; color: var(--ink-soft); margin-bottom: 7px; }
|
| 139 |
+
.sub-label .hint { text-transform: none; letter-spacing: .3px; color: var(--ink-faint); }
|
| 140 |
+
.warn { margin-top: 7px; font-size: 11px; color: var(--terra-deep); }
|
| 141 |
+
.excludes { display: flex; flex-direction: column; gap: 3px; }
|
| 142 |
+
.ex-row { display: flex; align-items: center; gap: 8px; font-size: 12.5px; color: var(--ink-soft); cursor: pointer; user-select: none; }
|
| 143 |
+
.ex-row .box { width: 14px; height: 14px; flex: none; border-radius: 4px; border: 1px solid var(--frame); background: #fbf6e8; position: relative; transition: background .16s, border-color .16s; }
|
| 144 |
+
.ex-row.on { color: var(--ink); }
|
| 145 |
+
.ex-row.on .box { background: var(--terra); border-color: var(--ink); }
|
| 146 |
+
.ex-row.on .box::after { content: 'β'; position: absolute; inset: 0; font-size: 10px; line-height: 13px; text-align: center; color: var(--ink); }
|
| 147 |
+
.exclude-note { margin-top: 9px; font-size: 10.5px; line-height: 1.5; color: var(--ink-faint); }
|
| 148 |
+
.tcard.docker-card .mid { word-break: break-word; font-size: 16px; }
|
| 149 |
+
.tcard.docker-card .pills { max-height: 54px; overflow: hidden; }
|
| 150 |
+
.tcard.docker-card .facts { margin-top: 9px; font-size: 10px; color: var(--ink-faint); letter-spacing: .5px; word-break: break-all; }
|
| 151 |
+
.btn-clear { width: 100%; cursor: pointer; background: transparent; border: 1px dashed var(--frame); border-radius: 10px; padding: 8px; color: var(--ink-faint); font-size: 11px; letter-spacing: 2px; text-transform: uppercase; transition: all .2s; }
|
| 152 |
+
.btn-clear:hover { border-color: var(--terra-deep); color: var(--terra-deep); }
|
| 153 |
+
|
| 154 |
+
/* section heads */
|
| 155 |
+
.crates-head { display: flex; align-items: baseline; justify-content: space-between; margin: 6px 0 16px; flex-wrap: wrap; gap: 10px; }
|
| 156 |
+
.crates-head h3 { font-family: var(--serif); font-weight: 400; font-size: 26px; }
|
| 157 |
+
.crates-head .sub { font-size: 11px; color: var(--ink-faint); letter-spacing: 1px; }
|
| 158 |
+
.sort-row { display: flex; gap: 8px; align-items: center; font-size: 11.5px; color: var(--ink-soft); }
|
| 159 |
+
.sort-row select { background: #fbf6e8; border: 1px solid var(--frame); border-radius: 8px; padding: 5px 9px; font-size: 11.5px; color: var(--ink); outline: none; }
|
| 160 |
+
|
| 161 |
+
/* ββ the spread (staff picks) ββ */
|
| 162 |
+
#spread { margin-bottom: 34px; }
|
| 163 |
+
.spread-row { display: grid; grid-template-columns: repeat(auto-fill, minmax(230px, 1fr)); gap: 20px; }
|
| 164 |
+
.pick { text-align: center; }
|
| 165 |
+
.pick .note { margin-top: 10px; font-family: var(--serif); font-style: italic; font-size: 13.5px; color: var(--ink-soft); line-height: 1.5; padding: 0 6px; }
|
| 166 |
+
.pick .by { margin-top: 4px; font-size: 10px; letter-spacing: 1.5px; text-transform: uppercase; color: var(--ink-faint); }
|
| 167 |
+
|
| 168 |
+
/* ββ tarot card ββ */
|
| 169 |
+
.grid { display: grid; grid-template-columns: repeat(auto-fill, minmax(240px, 1fr)); gap: 20px; }
|
| 170 |
+
.tcard { position: relative; cursor: pointer; border-radius: 12px; overflow: hidden;
|
| 171 |
+
background: var(--card); border: 1.5px solid var(--ink);
|
| 172 |
+
box-shadow: 3px 4px 0 rgba(58,47,34,.14);
|
| 173 |
+
transition: transform .22s var(--spring), box-shadow .25s var(--ease);
|
| 174 |
+
transform-style: preserve-3d; }
|
| 175 |
+
.tcard::after { /* inner frame */
|
| 176 |
+
content: ''; position: absolute; inset: 5px; border: 1px solid var(--frame); border-radius: 8px; pointer-events: none; }
|
| 177 |
+
.tcard:hover { transform: translateY(-4px) rotate(-.5deg); box-shadow: 5px 9px 0 rgba(58,47,34,.18); }
|
| 178 |
+
.tcard .sky { /* pastel top band */
|
| 179 |
+
background: var(--tc, var(--powder)); border-bottom: 1px solid var(--ink);
|
| 180 |
+
padding: 7px 14px 6px; display: flex; justify-content: space-between; align-items: center;
|
| 181 |
+
font-size: 10px; letter-spacing: 2px; color: var(--ink); }
|
| 182 |
+
.tcard .sky .num { font-weight: 600; }
|
| 183 |
+
.tcard .body { padding: 14px 15px 12px; text-align: center; }
|
| 184 |
+
.tcard .mid { font-family: var(--serif); font-size: 17.5px; line-height: 1.25; word-break: break-all; }
|
| 185 |
+
.tcard .author { font-size: 10.5px; color: var(--ink-faint); letter-spacing: 1px; margin-top: 3px; }
|
| 186 |
+
.tcard .meta { display: flex; gap: 9px; justify-content: center; margin-top: 9px; font-size: 10.5px; color: var(--ink-soft); flex-wrap: wrap; }
|
| 187 |
+
.tcard .meta .sc-match { color: var(--terra-deep); font-weight: 600; }
|
| 188 |
+
.tcard .pills { display: flex; flex-wrap: wrap; gap: 5px; justify-content: center; margin-top: 9px; min-height: 0; }
|
| 189 |
+
.tcard .pill { font-size: 9.5px; padding: 1.5px 8px; border-radius: 999px; border: 1px solid var(--ink); background: var(--pc, var(--sage)); color: var(--ink); }
|
| 190 |
+
.tcard .pill.hot { background: var(--terra); font-weight: 600; }
|
| 191 |
+
.tcard .lineage { margin-top: 8px; font-size: 10px; color: var(--ink-faint); font-style: italic; overflow: hidden; text-overflow: ellipsis; white-space: nowrap; }
|
| 192 |
+
.tcard .caption { border-top: 1px solid var(--ink); margin-top: 11px; padding: 7px 10px 8px;
|
| 193 |
+
font-size: 10px; letter-spacing: 2.5px; text-transform: uppercase; color: var(--ink);
|
| 194 |
+
background: var(--card2); position: relative; z-index: 0; }
|
| 195 |
+
.tcard.kernel-card .mid { word-break: break-word; }
|
| 196 |
+
.tcard.kernel-card .pills { max-height: 54px; overflow: hidden; }
|
| 197 |
+
.tcard.kernel-card .facts { margin-top: 9px; font-size: 10px; color: var(--ink-faint); letter-spacing: .5px; }
|
| 198 |
+
|
| 199 |
+
/* loader: ink-drawn spinning card */
|
| 200 |
+
.loader { display: none; margin: 46px auto; width: 46px; height: 66px; border-radius: 7px;
|
| 201 |
+
background: var(--card); border: 1.5px solid var(--ink); position: relative;
|
| 202 |
+
animation: flip 1s var(--ease) infinite; }
|
| 203 |
+
.loader::after { content: ''; position: absolute; inset: 5px; border: 1px solid var(--frame); border-radius: 4px; }
|
| 204 |
+
@keyframes flip { 0%{transform:rotateY(0)} 50%{transform:rotateY(180deg)} 100%{transform:rotateY(360deg)} }
|
| 205 |
+
.loading .loader { display: block; }
|
| 206 |
+
.loading .grid, .loading .spread-row { opacity: .35; }
|
| 207 |
+
|
| 208 |
+
.empty { text-align: center; padding: 60px 0; display: none; }
|
| 209 |
+
.empty .kao { font-size: 26px; color: var(--terra-deep); letter-spacing: 2px; }
|
| 210 |
+
.empty h4 { font-family: var(--serif); font-size: 27px; margin-top: 12px; }
|
| 211 |
+
.empty p { color: var(--ink-faint); margin-top: 8px; font-size: 13px; }
|
| 212 |
+
|
| 213 |
+
.btn-more { display: block; margin: 30px auto 0; cursor: pointer;
|
| 214 |
+
background: var(--card); border: 1.5px solid var(--ink); border-radius: 999px;
|
| 215 |
+
color: var(--ink); padding: 10px 30px; font-size: 11px; letter-spacing: 2.5px; text-transform: uppercase;
|
| 216 |
+
box-shadow: 3px 3px 0 rgba(58,47,34,.18);
|
| 217 |
+
transition: transform .18s var(--spring), box-shadow .18s var(--spring); }
|
| 218 |
+
.btn-more:hover { transform: translate(-1px,-2px); box-shadow: 4px 5px 0 rgba(58,47,34,.22); }
|
| 219 |
+
.btn-more:active { transform: translate(3px,3px); box-shadow: 0 0 0 rgba(58,47,34,.2); }
|
| 220 |
+
|
| 221 |
+
footer { border-top: 1px solid var(--line); padding: 22px 0 52px;
|
| 222 |
+
display: flex; justify-content: space-between; flex-wrap: wrap; gap: 12px;
|
| 223 |
+
font-size: 11.5px; color: var(--ink-faint); letter-spacing: .8px; }
|
| 224 |
+
footer .next { color: var(--marigold-deep); }
|
| 225 |
+
</style>
|
| 226 |
+
</head>
|
| 227 |
+
<body>
|
| 228 |
+
<div class="wrap">
|
| 229 |
+
|
| 230 |
+
<header id="hdr">
|
| 231 |
+
<div class="logo">
|
| 232 |
+
<div class="logo-disc"></div>
|
| 233 |
+
<div>
|
| 234 |
+
<h1>B-Sides.</h1>
|
| 235 |
+
<div class="tagline" id="tagline">deep search Β· indie & vintage LLMs</div>
|
| 236 |
+
</div>
|
| 237 |
+
</div>
|
| 238 |
+
<div class="head-right">
|
| 239 |
+
<span>π€ hugging face space</span><span class="dot">|</span><span id="head-count">β¦</span><span class="dot">|</span><span id="catalog-copy">2022 pressing</span>
|
| 240 |
+
</div>
|
| 241 |
+
</header>
|
| 242 |
+
|
| 243 |
+
<div class="main">
|
| 244 |
+
<aside class="rail" id="rail">
|
| 245 |
+
|
| 246 |
+
<div class="filter-stack" id="model-filters">
|
| 247 |
+
|
| 248 |
+
<div class="knob-group kg-exclude" id="exclude-group">
|
| 249 |
+
<h4><span class="swatch" style="background:var(--terra)"></span>Hidden by default</h4>
|
| 250 |
+
<div class="excludes" id="f-excludes"></div>
|
| 251 |
+
<p class="exclude-note">checked is hidden Β· uncheck to let them back into the crates</p>
|
| 252 |
+
</div>
|
| 253 |
+
|
| 254 |
+
<div class="knob-group kg-time">
|
| 255 |
+
<h4><span class="swatch" style="background:var(--rose)"></span>Time period</h4>
|
| 256 |
+
<div class="range-row">
|
| 257 |
+
<select id="f-month-from"></select><span>β</span><select id="f-month-to"></select>
|
| 258 |
+
</div>
|
| 259 |
+
</div>
|
| 260 |
+
|
| 261 |
+
<div class="knob-group kg-arch">
|
| 262 |
+
<h4><span class="swatch" style="background:var(--marigold)"></span>Architecture</h4>
|
| 263 |
+
<div class="chips" id="f-archs"></div>
|
| 264 |
+
</div>
|
| 265 |
+
|
| 266 |
+
<div class="knob-group kg-method">
|
| 267 |
+
<h4><span class="swatch" style="background:var(--sage)"></span>Training method</h4>
|
| 268 |
+
<div class="chips" id="f-methods"></div>
|
| 269 |
+
</div>
|
| 270 |
+
|
| 271 |
+
<div class="knob-group kg-lineage">
|
| 272 |
+
<h4><span class="swatch" style="background:var(--powder)"></span>Lineage</h4>
|
| 273 |
+
<select id="f-relation"><option value="">any relation</option></select>
|
| 274 |
+
<div style="height:8px"></div>
|
| 275 |
+
<input id="f-base" type="text" placeholder="base model containsβ¦">
|
| 276 |
+
</div>
|
| 277 |
+
|
| 278 |
+
<div class="knob-group kg-license">
|
| 279 |
+
<h4><span class="swatch" style="background:var(--terra)"></span>License</h4>
|
| 280 |
+
<select id="f-license"><option value="">any license</option></select>
|
| 281 |
+
</div>
|
| 282 |
+
|
| 283 |
+
<div class="knob-group kg-deep" id="deep-group">
|
| 284 |
+
<h4><span class="swatch" style="background:var(--lilac)"></span>Deep cuts <span class="soon" id="deep-soon" style="display:none">pressingβ¦</span></h4>
|
| 285 |
+
<div id="deep-filters">
|
| 286 |
+
<div class="sub-label">Model size (billions) <span class="hint">e.g. 1.5Bβ3.5B</span></div>
|
| 287 |
+
<div class="range-row">
|
| 288 |
+
<input id="f-pmin" type="number" placeholder="params β₯ (B)" step="0.1" min="0">
|
| 289 |
+
<input id="f-pmax" type="number" placeholder="β€ (B)" step="0.1" min="0">
|
| 290 |
+
</div>
|
| 291 |
+
<div class="warn" id="size-warn" style="display:none">minimum is above maximum</div>
|
| 292 |
+
<div style="height:8px"></div>
|
| 293 |
+
<input id="f-ctx" type="number" placeholder="context β₯ tokens">
|
| 294 |
+
<div style="height:8px"></div>
|
| 295 |
+
<input id="f-config" type="text" placeholder='config contains⦠"rope_scaling"'>
|
| 296 |
+
<div style="height:10px"></div>
|
| 297 |
+
<div class="chips" id="f-imports"></div>
|
| 298 |
+
</div>
|
| 299 |
+
<div class="toggle-row" id="f-moe"><span>MoE only</span><span class="switch"></span></div>
|
| 300 |
+
<div class="toggle-row" id="f-custom"><span>custom code only</span><span class="switch"></span></div>
|
| 301 |
+
</div>
|
| 302 |
+
|
| 303 |
+
</div>
|
| 304 |
+
|
| 305 |
+
<div class="filter-stack" id="kernel-filters" style="display:none">
|
| 306 |
+
<div class="knob-group kg-time">
|
| 307 |
+
<h4><span class="swatch" style="background:var(--rose)"></span>Time period</h4>
|
| 308 |
+
<div class="range-row">
|
| 309 |
+
<select id="k-month-from"></select><span>β</span><select id="k-month-to"></select>
|
| 310 |
+
</div>
|
| 311 |
+
</div>
|
| 312 |
+
|
| 313 |
+
<div class="knob-group kg-kernel" style="--chip-fill:var(--rose)">
|
| 314 |
+
<h4><span class="swatch" style="background:var(--rose)"></span>Downloads</h4>
|
| 315 |
+
<div class="range-row">
|
| 316 |
+
<input id="k-downloads-min" type="number" placeholder="downloads β₯" min="0">
|
| 317 |
+
<input id="k-downloads-max" type="number" placeholder="β€" min="0">
|
| 318 |
+
</div>
|
| 319 |
+
</div>
|
| 320 |
+
|
| 321 |
+
<div class="knob-group kg-kernel" style="--chip-fill:var(--marigold)">
|
| 322 |
+
<h4><span class="swatch" style="background:var(--marigold)"></span>Languages</h4>
|
| 323 |
+
<div class="chips" id="k-languages"></div>
|
| 324 |
+
</div>
|
| 325 |
+
|
| 326 |
+
<div class="knob-group kg-kernel" style="--chip-fill:var(--sage)">
|
| 327 |
+
<h4><span class="swatch" style="background:var(--sage)"></span>Accelerators</h4>
|
| 328 |
+
<div class="chips" id="k-accelerators"></div>
|
| 329 |
+
</div>
|
| 330 |
+
|
| 331 |
+
<div class="knob-group kg-kernel" style="--chip-fill:var(--powder)">
|
| 332 |
+
<h4><span class="swatch" style="background:var(--powder)"></span>CUDA architectures</h4>
|
| 333 |
+
<div class="chips" id="k-cuda-archs"></div>
|
| 334 |
+
</div>
|
| 335 |
+
|
| 336 |
+
<div class="knob-group kg-kernel" style="--chip-fill:var(--lilac)">
|
| 337 |
+
<h4><span class="swatch" style="background:var(--lilac)"></span>Dtypes</h4>
|
| 338 |
+
<div class="chips" id="k-dtypes"></div>
|
| 339 |
+
</div>
|
| 340 |
+
|
| 341 |
+
<div class="knob-group kg-kernel" style="--chip-fill:var(--terra)">
|
| 342 |
+
<h4><span class="swatch" style="background:var(--terra)"></span>Intrinsics</h4>
|
| 343 |
+
<div class="chips" id="k-intrinsics"></div>
|
| 344 |
+
</div>
|
| 345 |
+
|
| 346 |
+
<div class="knob-group kg-kernel" style="--chip-fill:var(--marigold)">
|
| 347 |
+
<h4><span class="swatch" style="background:var(--marigold)"></span>Torch operations</h4>
|
| 348 |
+
<div class="chips" id="k-torch-ops"></div>
|
| 349 |
+
</div>
|
| 350 |
+
|
| 351 |
+
<div class="knob-group kg-kernel" style="--chip-fill:var(--sage)">
|
| 352 |
+
<h4><span class="swatch" style="background:var(--sage)"></span>Torch versions</h4>
|
| 353 |
+
<div class="chips" id="k-torch-versions"></div>
|
| 354 |
+
</div>
|
| 355 |
+
|
| 356 |
+
<div class="knob-group kg-kernel">
|
| 357 |
+
<h4><span class="swatch" style="background:var(--powder)"></span>Author</h4>
|
| 358 |
+
<select id="k-author"><option value="">any author</option></select>
|
| 359 |
+
</div>
|
| 360 |
+
|
| 361 |
+
<div class="knob-group kg-kernel">
|
| 362 |
+
<h4><span class="swatch" style="background:var(--lilac)"></span>Minimum variants</h4>
|
| 363 |
+
<input id="k-variant-min" type="number" placeholder="variants β₯" min="0">
|
| 364 |
+
</div>
|
| 365 |
+
|
| 366 |
+
<div class="knob-group kg-kernel">
|
| 367 |
+
<h4><span class="swatch" style="background:var(--terra)"></span>Build facts</h4>
|
| 368 |
+
<div class="toggle-row" id="k-build"><span>has build directory</span><span class="switch"></span></div>
|
| 369 |
+
</div>
|
| 370 |
+
</div>
|
| 371 |
+
|
| 372 |
+
<div class="filter-stack" id="docker-filters" style="display:none">
|
| 373 |
+
<div class="knob-group kg-kernel" style="--chip-fill:var(--terra)">
|
| 374 |
+
<h4><span class="swatch" style="background:var(--terra)"></span>Artifact kind</h4>
|
| 375 |
+
<div class="toggle-row" id="d-archives"><span>include archives</span><span class="switch"></span></div>
|
| 376 |
+
<p class="exclude-note">recipes only by default Β· archives are tarballs, not build files</p>
|
| 377 |
+
</div>
|
| 378 |
+
|
| 379 |
+
<div class="knob-group kg-kernel" style="--chip-fill:var(--rose)">
|
| 380 |
+
<h4><span class="swatch" style="background:var(--rose)"></span>Repository type</h4>
|
| 381 |
+
<div class="chips" id="d-repo-types"></div>
|
| 382 |
+
</div>
|
| 383 |
+
|
| 384 |
+
<div class="knob-group kg-kernel" style="--chip-fill:var(--sage)">
|
| 385 |
+
<h4><span class="swatch" style="background:var(--sage)"></span>Accelerators</h4>
|
| 386 |
+
<div class="chips" id="d-accelerators"></div>
|
| 387 |
+
</div>
|
| 388 |
+
|
| 389 |
+
<div class="knob-group kg-kernel" style="--chip-fill:var(--powder)">
|
| 390 |
+
<h4><span class="swatch" style="background:var(--powder)"></span>CUDA versions</h4>
|
| 391 |
+
<div class="chips" id="d-cuda-versions"></div>
|
| 392 |
+
</div>
|
| 393 |
+
|
| 394 |
+
<div class="knob-group kg-kernel" style="--chip-fill:var(--terra)">
|
| 395 |
+
<h4><span class="swatch" style="background:var(--terra)"></span>ROCm versions</h4>
|
| 396 |
+
<div class="chips" id="d-rocm-versions"></div>
|
| 397 |
+
</div>
|
| 398 |
+
|
| 399 |
+
<div class="knob-group kg-kernel" style="--chip-fill:var(--marigold)">
|
| 400 |
+
<h4><span class="swatch" style="background:var(--marigold)"></span>Python versions</h4>
|
| 401 |
+
<div class="chips" id="d-python-versions"></div>
|
| 402 |
+
</div>
|
| 403 |
+
|
| 404 |
+
<div class="knob-group kg-kernel" style="--chip-fill:var(--lilac)">
|
| 405 |
+
<h4><span class="swatch" style="background:var(--lilac)"></span>Frameworks</h4>
|
| 406 |
+
<div class="chips" id="d-frameworks"></div>
|
| 407 |
+
</div>
|
| 408 |
+
|
| 409 |
+
<div class="knob-group kg-kernel" style="--chip-fill:var(--sage)">
|
| 410 |
+
<h4><span class="swatch" style="background:var(--sage)"></span>Base images</h4>
|
| 411 |
+
<div class="chips" id="d-base-images"></div>
|
| 412 |
+
</div>
|
| 413 |
+
|
| 414 |
+
<div class="knob-group kg-kernel" style="--chip-fill:var(--powder)">
|
| 415 |
+
<h4><span class="swatch" style="background:var(--powder)"></span>Operating systems</h4>
|
| 416 |
+
<div class="chips" id="d-operating-systems"></div>
|
| 417 |
+
</div>
|
| 418 |
+
|
| 419 |
+
<div class="knob-group kg-kernel" style="--chip-fill:var(--marigold)">
|
| 420 |
+
<h4><span class="swatch" style="background:var(--marigold)"></span>Package managers</h4>
|
| 421 |
+
<div class="chips" id="d-package-managers"></div>
|
| 422 |
+
</div>
|
| 423 |
+
|
| 424 |
+
<div class="knob-group kg-kernel" style="--chip-fill:var(--rose)">
|
| 425 |
+
<h4><span class="swatch" style="background:var(--rose)"></span>Services</h4>
|
| 426 |
+
<div class="chips" id="d-services"></div>
|
| 427 |
+
</div>
|
| 428 |
+
|
| 429 |
+
<div class="knob-group kg-kernel" style="--chip-fill:var(--lilac)">
|
| 430 |
+
<h4><span class="swatch" style="background:var(--lilac)"></span>Node versions</h4>
|
| 431 |
+
<div class="chips" id="d-node-versions"></div>
|
| 432 |
+
</div>
|
| 433 |
+
|
| 434 |
+
<!-- No exposed-ports control: ports are published as JSON integers and
|
| 435 |
+
the adapter only accepts string filter values, so any chip here
|
| 436 |
+
would match nothing. Needs a backend cast before it ships. -->
|
| 437 |
+
|
| 438 |
+
<div class="knob-group kg-kernel">
|
| 439 |
+
<h4><span class="swatch" style="background:var(--powder)"></span>Build facts</h4>
|
| 440 |
+
<div class="toggle-row" id="d-compose"><span>has compose file</span><span class="switch"></span></div>
|
| 441 |
+
<div class="toggle-row" id="d-devcontainer"><span>has devcontainer</span><span class="switch"></span></div>
|
| 442 |
+
<div class="toggle-row" id="d-multistage"><span>multi-stage build</span><span class="switch"></span></div>
|
| 443 |
+
</div>
|
| 444 |
+
</div>
|
| 445 |
+
|
| 446 |
+
<button class="btn-clear" id="clear">β clear the deck</button>
|
| 447 |
+
</aside>
|
| 448 |
+
|
| 449 |
+
<section>
|
| 450 |
+
<section class="hero">
|
| 451 |
+
<h2 id="hero-h">The models that never charted.<br><em>Dig anyway.</em></h2>
|
| 452 |
+
<div class="rule" id="rule">
|
| 453 |
+
<i style="background:var(--rose)"></i><i style="background:var(--marigold)"></i>
|
| 454 |
+
<i style="background:var(--sage)"></i><i style="background:var(--powder)"></i>
|
| 455 |
+
<i style="background:var(--lilac)"></i>
|
| 456 |
+
</div>
|
| 457 |
+
<div class="searchbar" id="term">
|
| 458 |
+
<span class="glyph">β¦</span>
|
| 459 |
+
<select class="corpus-select" id="corpus" aria-label="Search corpus">
|
| 460 |
+
<option value="models" selected>Models</option>
|
| 461 |
+
<option value="kernels">Kernels</option>
|
| 462 |
+
<option value="docker">Docker Images</option>
|
| 463 |
+
</select>
|
| 464 |
+
<input id="q" type="text" placeholder='search by meaning⦠"mamba hybrid trained on code"' autocomplete="off">
|
| 465 |
+
<button class="btn-dig" id="go">dig β΅</button>
|
| 466 |
+
</div>
|
| 467 |
+
<div class="status-line" id="status"></div>
|
| 468 |
+
<p class="promise" id="promise"><b id="hero-count">228,610</b> indie & vintage text-gen models Β· no chart-toppers, no quant reissues Β·
|
| 469 |
+
search by <b>meaning</b>, <b>time period</b>, <b>lineage</b>, or straight into the <b>config</b></p>
|
| 470 |
+
<p class="manifesto" id="manifesto">B-Sides index is more interesting than “models nobody uses.”
|
| 471 |
+
It maps the shadow infrastructure of open ML: abandoned experiments, regional
|
| 472 |
+
language work, architecture probes, creative utilities, and research releases that
|
| 473 |
+
conventional popularity rankings erase.</p>
|
| 474 |
+
</section>
|
| 475 |
+
|
| 476 |
+
<div id="spread" style="display:none">
|
| 477 |
+
<div class="crates-head">
|
| 478 |
+
<h3>The Spread.</h3>
|
| 479 |
+
<span class="sub">today's draw Β· rotates at midnight UTC</span>
|
| 480 |
+
</div>
|
| 481 |
+
<div class="spread-row" id="spread-row"></div>
|
| 482 |
+
</div>
|
| 483 |
+
|
| 484 |
+
<div class="crates-head" id="crates-head" style="display:none">
|
| 485 |
+
<h3 id="crate-title">The Reading.</h3>
|
| 486 |
+
<div class="sort-row">
|
| 487 |
+
sort
|
| 488 |
+
<select id="sort">
|
| 489 |
+
<option value="relevance">relevance</option>
|
| 490 |
+
<option value="newest" selected>newest</option>
|
| 491 |
+
<option value="oldest">oldest</option>
|
| 492 |
+
<option value="downloads">downloads</option>
|
| 493 |
+
<option value="likes">likes</option>
|
| 494 |
+
</select>
|
| 495 |
+
</div>
|
| 496 |
+
</div>
|
| 497 |
+
<div id="results-wrap">
|
| 498 |
+
<div class="grid" id="grid"></div>
|
| 499 |
+
<div class="loader"></div>
|
| 500 |
+
<div class="empty" id="empty">
|
| 501 |
+
<div class="kao">α¦(ΛΓ²_Γ³)α€οΈ»β¦β€β</div>
|
| 502 |
+
<h4 id="empty-title">The cards are silent.</h4>
|
| 503 |
+
<p id="empty-copy">no record answers to that. loosen a filter or flip the query.</p>
|
| 504 |
+
</div>
|
| 505 |
+
<button class="btn-more" id="more" style="display:none">β¬ dig deeper</button>
|
| 506 |
+
</div>
|
| 507 |
+
</section>
|
| 508 |
+
</div>
|
| 509 |
+
|
| 510 |
+
<footer>
|
| 511 |
+
<span id="footer-main">pressed by <a href="https://huggingface.co/juiceb0xc0de" target="_blank">juiceb0xc0de</a> Β· slop-free: no quant reissues, no chart-toppers</span>
|
| 512 |
+
<span class="next" id="footer-next">next pressings: 2024 catalog Β· config-deep filters</span>
|
| 513 |
+
</footer>
|
| 514 |
+
</div>
|
| 515 |
+
|
| 516 |
+
<script>
|
| 517 |
+
'use strict';
|
| 518 |
+
const $ = id => document.getElementById(id);
|
| 519 |
+
/* product defaults: the crates hide the majors and the quant reissues unless
|
| 520 |
+
the digger says otherwise. Checked in the UI means hidden from results. */
|
| 521 |
+
const DEFAULT_EXCLUDED_FAMILIES = ['llama','qwen','gemma','mistral','deepseek','phi','granite'];
|
| 522 |
+
const FAMILY_LABELS = {
|
| 523 |
+
llama: 'Llama', qwen: 'Qwen', gemma: 'Gemma', mistral: 'Mistral/Mixtral',
|
| 524 |
+
deepseek: 'DeepSeek', phi: 'Phi', granite: 'Granite',
|
| 525 |
+
};
|
| 526 |
+
/* mirrors CONTAINER_ARRAY_FILTERS server-side; each is its own AND group */
|
| 527 |
+
const DOCKER_ARRAY_FILTERS = ['base_images','accelerators','cuda_versions','rocm_versions',
|
| 528 |
+
'python_versions','node_versions','operating_systems','ports','services',
|
| 529 |
+
'package_managers','frameworks'];
|
| 530 |
+
const DOCKER_BOOL_FILTERS = ['has_compose','has_devcontainer','has_multistage'];
|
| 531 |
+
|
| 532 |
+
const state = {
|
| 533 |
+
corpus: 'models', page: 0, total: 0, busy: false, epoch: 0,
|
| 534 |
+
facets: { models: null, kernels: null, docker: null },
|
| 535 |
+
ready: { models: false, kernels: false, docker: false },
|
| 536 |
+
loading: { models: null, kernels: null, docker: null },
|
| 537 |
+
sel: {
|
| 538 |
+
models: {
|
| 539 |
+
archs: new Set(), methods: new Set(), imports: new Set(), moe: false, custom: false,
|
| 540 |
+
excludedFamilies: new Set(DEFAULT_EXCLUDED_FAMILIES),
|
| 541 |
+
excludeQuantizations: true,
|
| 542 |
+
},
|
| 543 |
+
kernels: {
|
| 544 |
+
languages: new Set(), accelerators: new Set(), cuda_archs: new Set(),
|
| 545 |
+
dtypes: new Set(), intrinsics: new Set(), torch_ops: new Set(),
|
| 546 |
+
torch_versions: new Set(), has_build_dir: false,
|
| 547 |
+
},
|
| 548 |
+
docker: {
|
| 549 |
+
repo_types: new Set(),
|
| 550 |
+
...Object.fromEntries(DOCKER_ARRAY_FILTERS.map(k => [k, new Set()])),
|
| 551 |
+
has_compose: false, has_devcontainer: false, has_multistage: false,
|
| 552 |
+
includeArchives: false,
|
| 553 |
+
},
|
| 554 |
+
},
|
| 555 |
+
};
|
| 556 |
+
|
| 557 |
+
const resultKey = {models:'models', kernels:'kernels', docker:'containers'};
|
| 558 |
+
const endpoint = (kind, corpus) => corpus === 'models'
|
| 559 |
+
? `/api/${kind}` : `/api/${kind}?corpus=${encodeURIComponent(corpus)}`;
|
| 560 |
+
/* card renderers are hoisted function declarations, so this map is safe here */
|
| 561 |
+
const CARD_RENDERERS = {models: tarotCard, kernels: kernelCard, docker: dockerCard};
|
| 562 |
+
const WARMING = {
|
| 563 |
+
models: 'warming the model cratesβ¦', kernels: 'warming the kernel cratesβ¦',
|
| 564 |
+
docker: 'warming the container cratesβ¦',
|
| 565 |
+
};
|
| 566 |
+
const IDLE_NOUNS = {
|
| 567 |
+
models: 'models in the crates', kernels: 'kernel repositories indexed',
|
| 568 |
+
docker: 'container artifacts indexed',
|
| 569 |
+
};
|
| 570 |
+
const RESULT_NOUNS = {models: 'models', kernels: 'kernels', docker: 'container artifacts'};
|
| 571 |
+
const HEAD_NOUNS = {models: 'models', kernels: 'kernels', docker: 'artifacts'};
|
| 572 |
+
const CRATE_TITLES = {
|
| 573 |
+
models: 'The Reading.', kernels: 'The Kernel Cut.', docker: 'The Build Sheet.',
|
| 574 |
+
};
|
| 575 |
+
|
| 576 |
+
const COPY = {
|
| 577 |
+
models: {
|
| 578 |
+
title: 'B-Sides β deep search for indie & vintage LLMs',
|
| 579 |
+
description: 'Search indie and vintage text-generation models on Hugging Face by meaning, time period, architecture, training method, lineage, and config internals.',
|
| 580 |
+
tagline: 'deep search Β· indie & vintage LLMs',
|
| 581 |
+
hero: 'The models that never charted.<br><em>Dig anyway.</em>',
|
| 582 |
+
placeholder: 'search by meaning⦠"mamba hybrid trained on code"',
|
| 583 |
+
promise: total => `<b id="hero-count">${total}</b> indie & vintage text-gen models Β· the majors and quant reissues are <b>hidden by default</b> β uncheck any of them in the rail to let them back in Β· search by <b>meaning</b>, <b>time period</b>, <b>lineage</b>, or straight into the <b>config</b>`,
|
| 584 |
+
manifesto: 'B-Sides index is more interesting than “models nobody uses.” It maps the shadow infrastructure of open ML: abandoned experiments, regional language work, architecture probes, creative utilities, and research releases that conventional popularity rankings erase.',
|
| 585 |
+
emptyTitle: 'The cards are silent.',
|
| 586 |
+
emptyCopy: 'no record answers to that. loosen a filter or flip the query.',
|
| 587 |
+
footer: 'pressed by <a href="https://huggingface.co/juiceb0xc0de" target="_blank">juiceb0xc0de</a> Β· slop-free: no quant reissues, no chart-toppers',
|
| 588 |
+
next: 'next pressings: 2024 catalog Β· config-deep filters',
|
| 589 |
+
catalog: '2022 pressing',
|
| 590 |
+
},
|
| 591 |
+
kernels: {
|
| 592 |
+
title: 'B-Sides β deep search for open ML kernels',
|
| 593 |
+
description: 'Search open ML kernel repositories on Hugging Face by meaning, language, accelerator, CUDA architecture, dtype, intrinsic, Torch operation, and build facts.',
|
| 594 |
+
tagline: 'deep search Β· open ML kernels',
|
| 595 |
+
hero: 'The kernels beneath the charts.<br><em>Dig deeper.</em>',
|
| 596 |
+
placeholder: 'search by meaning⦠"fused attention for Hopper"',
|
| 597 |
+
promise: total => `<b id="hero-count">${total}</b> open ML kernel repositories Β· search by <b>meaning</b>, <b>hardware path</b>, <b>dtype</b>, or <b>build facts</b>`,
|
| 598 |
+
manifesto: 'The kernel crates map the code beneath model releases: CUDA and Triton experiments, architecture-specific fast paths, fused operations, portability work, and optimization ideas that ordinary popularity rankings bury.',
|
| 599 |
+
emptyTitle: 'No kernel answered.',
|
| 600 |
+
emptyCopy: 'loosen a hardware filter or try another operation.',
|
| 601 |
+
footer: 'pressed by <a href="https://huggingface.co/juiceb0xc0de" target="_blank">juiceb0xc0de</a> Β· one repository per kernel result',
|
| 602 |
+
next: 'next pressing: build-graph filters',
|
| 603 |
+
catalog: 'kernel pressing',
|
| 604 |
+
},
|
| 605 |
+
docker: {
|
| 606 |
+
title: 'B-Sides β deep search for container recipes',
|
| 607 |
+
description: 'Search Docker and OCI build recipes on Hugging Face by meaning, base image, accelerator, CUDA or ROCm version, Python version, framework, service, and build facts.',
|
| 608 |
+
tagline: 'deep search Β· container recipes',
|
| 609 |
+
hero: 'The images nobody pushed.<br><em>Read the build.</em>',
|
| 610 |
+
placeholder: 'search by meaning⦠"Python 3.11 CUDA 12.8 PyTorch"',
|
| 611 |
+
promise: total => `<b id="hero-count">${total}</b> container artifacts Β· <b>recipes only</b> by default β flip <b>include archives</b> for the tarballs Β· search by <b>meaning</b>, <b>base image</b>, <b>accelerator</b>, or <b>build facts</b>`,
|
| 612 |
+
manifesto: 'The container crates hold the build instructions behind open ML: Dockerfiles, Compose stacks, and devcontainers that pin the exact CUDA, ROCm, Python, and framework versions a project actually ran on. It is the environment layer that model cards leave out.',
|
| 613 |
+
emptyTitle: 'No recipe answered.',
|
| 614 |
+
emptyCopy: 'loosen a build filter, or let the archives in.',
|
| 615 |
+
footer: 'pressed by <a href="https://huggingface.co/juiceb0xc0de" target="_blank">juiceb0xc0de</a> Β· one artifact per container result',
|
| 616 |
+
next: 'next pressing: layer-level build graphs',
|
| 617 |
+
catalog: 'container pressing',
|
| 618 |
+
},
|
| 619 |
+
};
|
| 620 |
+
|
| 621 |
+
/* pastel pairs: [top-band fill, pill fill] keyed by hash */
|
| 622 |
+
const PASTELS = [
|
| 623 |
+
['var(--rose)','var(--rose)'], ['var(--marigold)','var(--marigold)'],
|
| 624 |
+
['var(--sage)','var(--sage)'], ['var(--powder)','var(--powder)'],
|
| 625 |
+
['var(--lilac)','var(--lilac)'], ['var(--terra)','var(--terra)'],
|
| 626 |
+
];
|
| 627 |
+
const colorFor = s => PASTELS[[...s].reduce((a,c)=>a+c.charCodeAt(0),0) % PASTELS.length];
|
| 628 |
+
const fmtN = n => n >= 1e9 ? (n/1e9).toFixed(1).replace(/\.0$/,'')+'B'
|
| 629 |
+
: n >= 1e6 ? (n/1e6).toFixed(1).replace(/\.0$/,'')+'M'
|
| 630 |
+
: n >= 1e3 ? (n/1e3).toFixed(1).replace(/\.0$/,'')+'K' : String(n);
|
| 631 |
+
const esc = s => String(s).replace(/[&<>"']/g, c => ({'&':'&','<':'<','>':'>','"':'"',"'":'''}[c]));
|
| 632 |
+
const ROMAN = n => { // small roman numerals for card numbers
|
| 633 |
+
const map = [[1000,'M'],[900,'CM'],[500,'D'],[400,'CD'],[100,'C'],[90,'XC'],[50,'L'],[40,'XL'],[10,'X'],[9,'IX'],[5,'V'],[4,'IV'],[1,'I']];
|
| 634 |
+
let out = ''; n = (n % 3999) + 1;
|
| 635 |
+
for (const [v, r] of map) while (n >= v) { out += r; n -= v; }
|
| 636 |
+
return out;
|
| 637 |
+
};
|
| 638 |
+
|
| 639 |
+
function buildModelReq() {
|
| 640 |
+
const s = state.sel.models;
|
| 641 |
+
const pB = v => v ? Math.round(parseFloat(v) * 1e9) : null;
|
| 642 |
+
return {
|
| 643 |
+
query: $('q').value.trim(),
|
| 644 |
+
month_from: $('f-month-from').value || null,
|
| 645 |
+
month_to: $('f-month-to').value || null,
|
| 646 |
+
archs: [...s.archs], methods: [...s.methods], imports: [...s.imports],
|
| 647 |
+
relation: $('f-relation').value || null,
|
| 648 |
+
base_model: $('f-base').value.trim() || null,
|
| 649 |
+
license: $('f-license').value || null,
|
| 650 |
+
params_min: pB($('f-pmin').value), params_max: pB($('f-pmax').value),
|
| 651 |
+
context_min: $('f-ctx').value ? parseInt($('f-ctx').value) : null,
|
| 652 |
+
config_contains: $('f-config').value.trim() || null,
|
| 653 |
+
moe_only: s.moe, custom_code_only: s.custom,
|
| 654 |
+
exclude_families: [...state.sel.models.excludedFamilies],
|
| 655 |
+
exclude_quantizations: state.sel.models.excludeQuantizations,
|
| 656 |
+
sort: $('sort').value, page: state.page,
|
| 657 |
+
};
|
| 658 |
+
}
|
| 659 |
+
|
| 660 |
+
/* an inverted size range can never match anything β say so instead of asking
|
| 661 |
+
the backend for the empty set */
|
| 662 |
+
function sizeRangeInvalid() {
|
| 663 |
+
const min = parseFloat($('f-pmin').value);
|
| 664 |
+
const max = parseFloat($('f-pmax').value);
|
| 665 |
+
const bad = Number.isFinite(min) && Number.isFinite(max) && min > max;
|
| 666 |
+
$('size-warn').style.display = bad ? '' : 'none';
|
| 667 |
+
return bad;
|
| 668 |
+
}
|
| 669 |
+
|
| 670 |
+
function buildDockerReq() {
|
| 671 |
+
const s = state.sel.docker;
|
| 672 |
+
const filters = {};
|
| 673 |
+
const put = (key, values) => { if (values.length) filters[key] = values; };
|
| 674 |
+
// Recipes are the product default; the archive toggle drops the constraint
|
| 675 |
+
// rather than widening it to a second kind.
|
| 676 |
+
if (!s.includeArchives) filters.artifact_kinds = ['recipe'];
|
| 677 |
+
put('repo_types', [...s.repo_types]);
|
| 678 |
+
for (const key of DOCKER_ARRAY_FILTERS) put(key, [...s[key]]);
|
| 679 |
+
for (const key of DOCKER_BOOL_FILTERS) if (s[key]) filters[key] = true;
|
| 680 |
+
return {
|
| 681 |
+
query: $('q').value.trim(), sort: $('sort').value,
|
| 682 |
+
page: state.page, filters,
|
| 683 |
+
};
|
| 684 |
+
}
|
| 685 |
+
|
| 686 |
+
function buildKernelReq() {
|
| 687 |
+
const s = state.sel.kernels;
|
| 688 |
+
const filters = {};
|
| 689 |
+
const put = (key, value) => {
|
| 690 |
+
if (value !== '' && value != null && (!Array.isArray(value) || value.length))
|
| 691 |
+
filters[key] = value;
|
| 692 |
+
};
|
| 693 |
+
put('month_from', $('k-month-from').value);
|
| 694 |
+
put('month_to', $('k-month-to').value);
|
| 695 |
+
put('author', $('k-author').value);
|
| 696 |
+
put('downloads_min', $('k-downloads-min').value ? parseInt($('k-downloads-min').value) : null);
|
| 697 |
+
put('downloads_max', $('k-downloads-max').value ? parseInt($('k-downloads-max').value) : null);
|
| 698 |
+
put('variant_count_min', $('k-variant-min').value ? parseInt($('k-variant-min').value) : null);
|
| 699 |
+
for (const key of ['languages','accelerators','cuda_archs','dtypes','intrinsics','torch_ops','torch_versions'])
|
| 700 |
+
put(key, [...s[key]]);
|
| 701 |
+
if (s.has_build_dir) filters.has_build_dir = true;
|
| 702 |
+
return {
|
| 703 |
+
query: $('q').value.trim(), sort: $('sort').value,
|
| 704 |
+
page: state.page, filters,
|
| 705 |
+
};
|
| 706 |
+
}
|
| 707 |
+
|
| 708 |
+
const REQ_BUILDERS = {models: buildModelReq, kernels: buildKernelReq, docker: buildDockerReq};
|
| 709 |
+
const buildReq = () => REQ_BUILDERS[state.corpus]();
|
| 710 |
+
|
| 711 |
+
// default exclusions are product defaults, not user intent β they leave the
|
| 712 |
+
// landing state pristine, same as the default Docker recipe constraint.
|
| 713 |
+
const modelIsPristine = r => !r.query && !r.month_from && !r.month_to && !r.archs.length
|
| 714 |
+
&& !r.methods.length && !r.imports.length && !r.relation && !r.base_model
|
| 715 |
+
&& !r.license && r.params_min == null && r.params_max == null
|
| 716 |
+
&& r.context_min == null && !r.config_contains && !r.moe_only && !r.custom_code_only;
|
| 717 |
+
const kernelIsPristine = r => !r.query && Object.keys(r.filters).length === 0;
|
| 718 |
+
const dockerIsPristine = r => {
|
| 719 |
+
if (r.query) return false;
|
| 720 |
+
const keys = Object.keys(r.filters);
|
| 721 |
+
return keys.length === 0 || (keys.length === 1 && keys[0] === 'artifact_kinds');
|
| 722 |
+
};
|
| 723 |
+
const PRISTINE = {models: modelIsPristine, kernels: kernelIsPristine, docker: dockerIsPristine};
|
| 724 |
+
const isPristine = r => PRISTINE[state.corpus](r);
|
| 725 |
+
|
| 726 |
+
function tarotCard(m) {
|
| 727 |
+
const archTag = m.model_type || m.tags.find(t => state.facets.models._archset.has(t)) || '';
|
| 728 |
+
const [band] = colorFor(archTag || m.model_id);
|
| 729 |
+
const method = ['grpo','dpo','orpo','kto','ppo','sft','lora','merge'].find(t => m.tags.includes(t));
|
| 730 |
+
|
| 731 |
+
const pills = [];
|
| 732 |
+
for (const imp of (m.code_imports || []).filter(i => ['flash_attn','causal_conv1d','mamba_ssm','triton','fla'].includes(i)))
|
| 733 |
+
pills.push(`<span class="pill hot">${esc(imp)}</span>`);
|
| 734 |
+
if (m.gated) pills.push(`<span class="pill" style="--pc:var(--lilac)">gated</span>`);
|
| 735 |
+
if (m.tags.includes('unsloth')) pills.push(`<span class="pill" style="--pc:var(--sage)">unsloth</span>`);
|
| 736 |
+
|
| 737 |
+
const meta = [`<span>${esc(m.month || '')}</span>`];
|
| 738 |
+
if (m.params) meta.push(`<span>${fmtN(m.params)} params</span>`);
|
| 739 |
+
if (m.context_len) meta.push(`<span>${fmtN(m.context_len)} ctx</span>`);
|
| 740 |
+
meta.push(`<span>β${fmtN(m.downloads||0)} β₯${fmtN(m.likes||0)}</span>`);
|
| 741 |
+
if (m.score != null) meta.push(`<span class="sc-match">${(m.score*100).toFixed(0)}%</span>`);
|
| 742 |
+
|
| 743 |
+
const lineage = m.base_model
|
| 744 |
+
? `<div class="lineage">${esc(m.relation || 'b-side')} of ${esc(m.base_model)}</div>` : '';
|
| 745 |
+
const caption = [archTag, method].filter(Boolean).join(' Β· ') || 'text generation';
|
| 746 |
+
|
| 747 |
+
const [author, ...rest] = m.model_id.split('/');
|
| 748 |
+
return `<div class="tcard" style="--tc:${band}" data-href="https://huggingface.co/${encodeURIComponent(author)}/${rest.map(encodeURIComponent).join('/')}">
|
| 749 |
+
<div class="sky"><span class="num">β ${ROMAN(m.row)}</span><span>${esc(m.month || '')}</span></div>
|
| 750 |
+
<div class="body">
|
| 751 |
+
<div class="mid">${esc(rest.join('/') || m.model_id)}</div>
|
| 752 |
+
<div class="author">${esc(author)}</div>
|
| 753 |
+
<div class="meta">${meta.join('')}</div>
|
| 754 |
+
${pills.length ? `<div class="pills">${pills.join('')}</div>` : ''}
|
| 755 |
+
${lineage}
|
| 756 |
+
</div>
|
| 757 |
+
<div class="caption">${esc(caption)} .</div>
|
| 758 |
+
</div>`;
|
| 759 |
+
}
|
| 760 |
+
|
| 761 |
+
function kernelCard(k) {
|
| 762 |
+
const selected = state.sel.kernels;
|
| 763 |
+
const [authorFromId, ...rest] = k.repo_id.split('/');
|
| 764 |
+
const author = k.author || authorFromId;
|
| 765 |
+
const name = rest.join('/') || k.repo_id;
|
| 766 |
+
const [band] = colorFor((k.accelerators || [])[0] || k.repo_id);
|
| 767 |
+
const preferSelected = (values, set, limit) => [...(values || [])]
|
| 768 |
+
.sort((a, b) => Number(set.has(b)) - Number(set.has(a))).slice(0, limit);
|
| 769 |
+
|
| 770 |
+
const pills = [];
|
| 771 |
+
for (const value of [...(k.languages || []), ...(k.accelerators || [])].slice(0, 4))
|
| 772 |
+
pills.push(`<span class="pill hot">${esc(value)}</span>`);
|
| 773 |
+
for (const value of (k.dtypes || []).slice(0, 2))
|
| 774 |
+
pills.push(`<span class="pill" style="--pc:var(--lilac)">${esc(value)}</span>`);
|
| 775 |
+
for (const value of preferSelected(k.intrinsics, selected.intrinsics, 2))
|
| 776 |
+
pills.push(`<span class="pill" style="--pc:var(--terra)">${esc(value)}</span>`);
|
| 777 |
+
for (const value of preferSelected(k.torch_ops, selected.torch_ops, 2))
|
| 778 |
+
pills.push(`<span class="pill" style="--pc:var(--sage)">${esc(value)}</span>`);
|
| 779 |
+
|
| 780 |
+
const meta = [`<span>${esc(k.month || '')}</span>`,
|
| 781 |
+
`<span>β${fmtN(k.downloads||0)} β₯${fmtN(k.likes||0)}</span>`];
|
| 782 |
+
if (k.score != null) meta.push(`<span class="sc-match">${(k.score*100).toFixed(0)}%</span>`);
|
| 783 |
+
const variants = `${fmtN(k.variant_count || 0)} variant${k.variant_count === 1 ? '' : 's'}`;
|
| 784 |
+
const sources = `${fmtN(k.source_files || 0)} source file${k.source_files === 1 ? '' : 's'}`;
|
| 785 |
+
const href = `https://huggingface.co/${k.repo_id.split('/').map(encodeURIComponent).join('/')}`;
|
| 786 |
+
|
| 787 |
+
return `<div class="tcard kernel-card" style="--tc:${band}" data-href="${href}">
|
| 788 |
+
<div class="sky"><span class="num">β ${ROMAN(k.row)}</span><span>${esc(k.month || '')}</span></div>
|
| 789 |
+
<div class="body">
|
| 790 |
+
<div class="mid">${esc(name)}</div>
|
| 791 |
+
<div class="author">${esc(author)}</div>
|
| 792 |
+
<div class="meta">${meta.join('')}</div>
|
| 793 |
+
${pills.length ? `<div class="pills">${pills.slice(0, 10).join('')}</div>` : ''}
|
| 794 |
+
<div class="facts">${variants} Β· ${sources}</div>
|
| 795 |
+
</div>
|
| 796 |
+
<div class="caption">kernel repository .</div>
|
| 797 |
+
</div>`;
|
| 798 |
+
}
|
| 799 |
+
|
| 800 |
+
/* Buckets have no git revisions, and recipe rows carry the synthetic
|
| 801 |
+
`<recipe>` path rather than a real file β neither has a valid blob URL, so
|
| 802 |
+
those land on the repository page instead of a guaranteed 404. */
|
| 803 |
+
function containerHref(c) {
|
| 804 |
+
const repoType = c.repo_type || 'model';
|
| 805 |
+
const path = c.artifact_path || '';
|
| 806 |
+
const repo = String(c.repo_id || '').split('/').map(encodeURIComponent).join('/');
|
| 807 |
+
if (repoType === 'bucket') return `https://huggingface.co/buckets/${repo}`;
|
| 808 |
+
const prefix = repoType === 'space' ? 'spaces/' : repoType === 'dataset' ? 'datasets/' : '';
|
| 809 |
+
const landing = `https://huggingface.co/${prefix}${repo}`;
|
| 810 |
+
if (!c.sha || !path || path === '<recipe>') return landing;
|
| 811 |
+
const file = path.split('/').map(encodeURIComponent).join('/');
|
| 812 |
+
return `${landing}/blob/${encodeURIComponent(c.sha)}/${file}`;
|
| 813 |
+
}
|
| 814 |
+
|
| 815 |
+
function dockerCard(c) {
|
| 816 |
+
const selected = state.sel.docker;
|
| 817 |
+
const [authorFromId, ...rest] = String(c.repo_id || '').split('/');
|
| 818 |
+
const author = c.author || authorFromId;
|
| 819 |
+
const name = rest.join('/') || c.repo_id || '';
|
| 820 |
+
const path = c.artifact_path || '';
|
| 821 |
+
const kind = c.artifact_kind || 'recipe';
|
| 822 |
+
const month = (c.created_at || '').slice(0, 7);
|
| 823 |
+
const [band] = colorFor((c.accelerators || [])[0] || (c.base_images || [])[0] || name);
|
| 824 |
+
const preferSelected = (values, set, limit) => [...(values || [])]
|
| 825 |
+
.sort((a, b) => Number(set.has(b)) - Number(set.has(a))).slice(0, limit);
|
| 826 |
+
|
| 827 |
+
const pills = [];
|
| 828 |
+
for (const value of (c.accelerators || []).slice(0, 2))
|
| 829 |
+
pills.push(`<span class="pill hot">${esc(value)}</span>`);
|
| 830 |
+
for (const value of preferSelected(c.cuda_versions, selected.cuda_versions, 2))
|
| 831 |
+
pills.push(`<span class="pill" style="--pc:var(--powder)">cuda ${esc(value)}</span>`);
|
| 832 |
+
for (const value of preferSelected(c.rocm_versions, selected.rocm_versions, 1))
|
| 833 |
+
pills.push(`<span class="pill" style="--pc:var(--terra)">rocm ${esc(value)}</span>`);
|
| 834 |
+
for (const value of preferSelected(c.python_versions, selected.python_versions, 2))
|
| 835 |
+
pills.push(`<span class="pill" style="--pc:var(--marigold)">py ${esc(value)}</span>`);
|
| 836 |
+
for (const value of preferSelected(c.frameworks, selected.frameworks, 3))
|
| 837 |
+
pills.push(`<span class="pill" style="--pc:var(--sage)">${esc(value)}</span>`);
|
| 838 |
+
if (c.has_compose) pills.push(`<span class="pill" style="--pc:var(--lilac)">compose</span>`);
|
| 839 |
+
if (c.has_devcontainer) pills.push(`<span class="pill" style="--pc:var(--lilac)">devcontainer</span>`);
|
| 840 |
+
if (c.has_multistage) pills.push(`<span class="pill" style="--pc:var(--lilac)">multi-stage</span>`);
|
| 841 |
+
|
| 842 |
+
const meta = [`<span>${esc(month)}</span>`,
|
| 843 |
+
`<span>β${fmtN(c.downloads||0)} β₯${fmtN(c.likes||0)}</span>`];
|
| 844 |
+
if (c.score != null) meta.push(`<span class="sc-match">${(c.score*100).toFixed(0)}%</span>`);
|
| 845 |
+
|
| 846 |
+
const base = (c.base_images || [])[0];
|
| 847 |
+
const facts = [base ? `from ${esc(base)}` : '',
|
| 848 |
+
path && path !== '<recipe>' ? esc(path) : esc(kind)].filter(Boolean).join(' Β· ');
|
| 849 |
+
const caption = [kind, c.repo_type].filter(Boolean).join(' Β· ') || 'container artifact';
|
| 850 |
+
|
| 851 |
+
return `<div class="tcard docker-card" style="--tc:${band}" data-href="${esc(containerHref(c))}">
|
| 852 |
+
<div class="sky"><span class="num">β ${ROMAN(c.row)}</span><span>${esc(month)}</span></div>
|
| 853 |
+
<div class="body">
|
| 854 |
+
<div class="mid">${esc(name)}</div>
|
| 855 |
+
<div class="author">${esc(author)}</div>
|
| 856 |
+
<div class="meta">${meta.join('')}</div>
|
| 857 |
+
${pills.length ? `<div class="pills">${pills.slice(0, 10).join('')}</div>` : ''}
|
| 858 |
+
<div class="facts">${facts}</div>
|
| 859 |
+
</div>
|
| 860 |
+
<div class="caption">${esc(caption)} .</div>
|
| 861 |
+
</div>`;
|
| 862 |
+
}
|
| 863 |
+
|
| 864 |
+
function wireCards(scope) {
|
| 865 |
+
const fresh = [...scope.querySelectorAll('.tcard:not(.wired)')];
|
| 866 |
+
fresh.forEach(el => {
|
| 867 |
+
el.classList.add('wired');
|
| 868 |
+
el.addEventListener('click', () => window.open(el.dataset.href, '_blank'));
|
| 869 |
+
});
|
| 870 |
+
if (window.VanillaTilt) VanillaTilt.init(fresh, { max: 4, speed: 900, scale: 1.015, gyroscope: false });
|
| 871 |
+
if (window.gsap) gsap.from(fresh, { y: 16, opacity: 0, duration: .45, stagger: 0.04, ease: 'power3.out', clearProps: 'opacity,transform' });
|
| 872 |
+
}
|
| 873 |
+
|
| 874 |
+
// boot window: the backend returns 503 while the ~1.4 GB index streams in off
|
| 875 |
+
// disk β retry politely instead of showing a broken page.
|
| 876 |
+
async function getJSON(url, opts, tries = 60, delayMs = 1000) {
|
| 877 |
+
for (let i = 0; ; i++) {
|
| 878 |
+
const r = await fetch(url, opts);
|
| 879 |
+
if (r.ok) return r.json();
|
| 880 |
+
if (r.status !== 503 || i >= tries)
|
| 881 |
+
throw new Error((await r.json().catch(() => ({}))).detail
|
| 882 |
+
|| `request failed (${r.status})`);
|
| 883 |
+
await new Promise(res => setTimeout(res, delayMs));
|
| 884 |
+
}
|
| 885 |
+
}
|
| 886 |
+
|
| 887 |
+
async function loadSpread() {
|
| 888 |
+
try {
|
| 889 |
+
const d = await getJSON('/api/picks');
|
| 890 |
+
if (!d.picks.length) return;
|
| 891 |
+
$('spread-row').innerHTML = d.picks.map(p => `
|
| 892 |
+
<div class="pick">
|
| 893 |
+
${tarotCard(p)}
|
| 894 |
+
<div class="note">β${esc(p.note)}β</div>
|
| 895 |
+
<div class="by">β ${esc(p.picked_by)}</div>
|
| 896 |
+
</div>`).join('');
|
| 897 |
+
wireCards($('spread-row'));
|
| 898 |
+
if (state.corpus === 'models') $('spread').style.display = '';
|
| 899 |
+
} catch (e) { /* picks are optional */ }
|
| 900 |
+
}
|
| 901 |
+
|
| 902 |
+
function idle() {
|
| 903 |
+
/* landing state: search engine at rest β spread only, no listing */
|
| 904 |
+
$('grid').innerHTML = '';
|
| 905 |
+
$('crates-head').style.display = 'none';
|
| 906 |
+
$('empty').style.display = 'none';
|
| 907 |
+
$('more').style.display = 'none';
|
| 908 |
+
$('spread').style.display = state.corpus === 'models' && $('spread-row').children.length ? '' : 'none';
|
| 909 |
+
const facets = state.facets[state.corpus];
|
| 910 |
+
$('status').innerHTML = facets
|
| 911 |
+
? `<b>${facets.total.toLocaleString()}</b> ${IDLE_NOUNS[state.corpus]}` : '';
|
| 912 |
+
}
|
| 913 |
+
|
| 914 |
+
async function search(append=false) {
|
| 915 |
+
if (state.busy) return;
|
| 916 |
+
if (!state.ready[state.corpus]) {
|
| 917 |
+
$('status').textContent = WARMING[state.corpus];
|
| 918 |
+
return;
|
| 919 |
+
}
|
| 920 |
+
if (state.corpus === 'models' && sizeRangeInvalid()) return;
|
| 921 |
+
const reqCheck = buildReq();
|
| 922 |
+
if (isPristine(reqCheck)) { idle(); return; }
|
| 923 |
+
const corpus = state.corpus;
|
| 924 |
+
const epoch = state.epoch;
|
| 925 |
+
state.busy = true;
|
| 926 |
+
if (!append) { state.page = 0; $('grid').innerHTML = ''; }
|
| 927 |
+
$('results-wrap').parentElement.classList.add('loading');
|
| 928 |
+
$('empty').style.display = 'none';
|
| 929 |
+
try {
|
| 930 |
+
const req = buildReq();
|
| 931 |
+
const data = await getJSON(endpoint('search', corpus), { method: 'POST',
|
| 932 |
+
headers: {'Content-Type':'application/json'}, body: JSON.stringify(req) }, 60, 1000);
|
| 933 |
+
if (corpus !== state.corpus || epoch !== state.epoch) return;
|
| 934 |
+
state.total = data.total;
|
| 935 |
+
const rows = data[resultKey[corpus]] || [];
|
| 936 |
+
const html = rows.map(CARD_RENDERERS[corpus]).join('');
|
| 937 |
+
if (append) $('grid').insertAdjacentHTML('beforeend', html);
|
| 938 |
+
else $('grid').innerHTML = html;
|
| 939 |
+
wireCards($('grid'));
|
| 940 |
+
|
| 941 |
+
$('spread').style.display = 'none';
|
| 942 |
+
$('crates-head').style.display = '';
|
| 943 |
+
const noun = RESULT_NOUNS[corpus];
|
| 944 |
+
$('status').innerHTML = `<b>${data.total.toLocaleString()}</b> ${noun}` +
|
| 945 |
+
(req.query && req.sort === 'relevance' ? ` Β· matched by meaning to β${esc(req.query)}β` : ' match your filters');
|
| 946 |
+
$('empty').style.display = data.total === 0 ? 'block' : 'none';
|
| 947 |
+
$('more').style.display = $('grid').children.length < data.total ? 'block' : 'none';
|
| 948 |
+
} catch (e) {
|
| 949 |
+
if (corpus === state.corpus && epoch === state.epoch)
|
| 950 |
+
$('status').innerHTML = `β ${esc(e.message)}`;
|
| 951 |
+
} finally {
|
| 952 |
+
if (corpus === state.corpus && epoch === state.epoch) {
|
| 953 |
+
$('results-wrap').parentElement.classList.remove('loading');
|
| 954 |
+
state.busy = false;
|
| 955 |
+
}
|
| 956 |
+
}
|
| 957 |
+
}
|
| 958 |
+
|
| 959 |
+
function chipbar(el, items, set, max=14) {
|
| 960 |
+
const render = (expanded) => {
|
| 961 |
+
const list = expanded ? items : items.slice(0, max);
|
| 962 |
+
el.innerHTML = list.map(i =>
|
| 963 |
+
`<span class="chip${set.has(i.v)?' on':''}" data-v="${esc(i.v)}">${esc(i.v)}<span class="n">${fmtN(i.n)}</span></span>`
|
| 964 |
+
).join('') + (items.length > max
|
| 965 |
+
? `<span class="chip more-toggle">${expanded ? 'β less' : `+ ${items.length-max} more`}</span>` : '');
|
| 966 |
+
el.querySelectorAll('.chip').forEach(c => c.onclick = () => {
|
| 967 |
+
if (c.classList.contains('more-toggle')) return render(!expanded);
|
| 968 |
+
const v = c.dataset.v;
|
| 969 |
+
set.has(v) ? set.delete(v) : set.add(v);
|
| 970 |
+
c.classList.toggle('on');
|
| 971 |
+
search();
|
| 972 |
+
});
|
| 973 |
+
};
|
| 974 |
+
render(false);
|
| 975 |
+
}
|
| 976 |
+
|
| 977 |
+
function monthOptions(from, to, months) {
|
| 978 |
+
$(from).innerHTML = '<option value="">from the start</option>' +
|
| 979 |
+
months.map(m=>`<option>${m}</option>`).join('');
|
| 980 |
+
$(to).innerHTML = '<option value="">to the latest</option>' +
|
| 981 |
+
months.map(m=>`<option>${m}</option>`).join('');
|
| 982 |
+
}
|
| 983 |
+
|
| 984 |
+
function populateModelFacets(f) {
|
| 985 |
+
f._archset = new Set(f.archs.map(a => a.v));
|
| 986 |
+
monthOptions('f-month-from', 'f-month-to', f.months);
|
| 987 |
+
chipbar($('f-archs'), f.archs, state.sel.models.archs, 6);
|
| 988 |
+
chipbar($('f-methods'), f.methods, state.sel.models.methods, 6);
|
| 989 |
+
$('f-relation').innerHTML = '<option value="">any relation</option>' +
|
| 990 |
+
f.relations.map(r=>`<option value="${esc(r.v)}">${esc(r.v)} (${fmtN(r.n)})</option>`).join('');
|
| 991 |
+
$('f-license').innerHTML = '<option value="">any license</option>' +
|
| 992 |
+
f.licenses.map(l=>`<option value="${esc(l.v)}">${esc(l.v)} (${fmtN(l.n)})</option>`).join('');
|
| 993 |
+
if (f.enriched) {
|
| 994 |
+
if (f.imports.length) chipbar($('f-imports'), f.imports, state.sel.models.imports, 10);
|
| 995 |
+
} else {
|
| 996 |
+
$('deep-soon').style.display = '';
|
| 997 |
+
$('deep-filters').style.display = 'none';
|
| 998 |
+
}
|
| 999 |
+
}
|
| 1000 |
+
|
| 1001 |
+
function populateKernelFacets(f) {
|
| 1002 |
+
monthOptions('k-month-from', 'k-month-to', f.months);
|
| 1003 |
+
for (const [key, id, max] of [
|
| 1004 |
+
['languages','k-languages',10], ['accelerators','k-accelerators',8],
|
| 1005 |
+
['cuda_archs','k-cuda-archs',10], ['dtypes','k-dtypes',10],
|
| 1006 |
+
['intrinsics','k-intrinsics',10], ['torch_ops','k-torch-ops',10],
|
| 1007 |
+
['torch_versions','k-torch-versions',10],
|
| 1008 |
+
]) chipbar($(id), f[key] || [], state.sel.kernels[key], max);
|
| 1009 |
+
$('k-author').innerHTML = '<option value="">any author</option>' +
|
| 1010 |
+
(f.authors || []).map(a=>`<option value="${esc(a.v)}">${esc(a.v)} (${fmtN(a.n)})</option>`).join('');
|
| 1011 |
+
}
|
| 1012 |
+
|
| 1013 |
+
/* checked means hidden. These are product defaults, so they render on; the
|
| 1014 |
+
list is fixed by DEFAULT_EXCLUDED_FAMILIES and stays visible once unchecked. */
|
| 1015 |
+
function renderExclusions() {
|
| 1016 |
+
const s = state.sel.models;
|
| 1017 |
+
const rows = [['__quant', 'Quantizations', s.excludeQuantizations]].concat(
|
| 1018 |
+
DEFAULT_EXCLUDED_FAMILIES.map(f => [f, FAMILY_LABELS[f] || f, s.excludedFamilies.has(f)]));
|
| 1019 |
+
$('f-excludes').innerHTML = rows.map(([key, label, on]) =>
|
| 1020 |
+
`<span class="ex-row${on ? ' on' : ''}" data-k="${esc(key)}"><span class="box"></span>${esc(label)}</span>`
|
| 1021 |
+
).join('');
|
| 1022 |
+
$('f-excludes').querySelectorAll('.ex-row').forEach(row => row.onclick = () => {
|
| 1023 |
+
const key = row.dataset.k;
|
| 1024 |
+
if (key === '__quant') s.excludeQuantizations = !s.excludeQuantizations;
|
| 1025 |
+
else s.excludedFamilies.has(key)
|
| 1026 |
+
? s.excludedFamilies.delete(key) : s.excludedFamilies.add(key);
|
| 1027 |
+
row.classList.toggle('on');
|
| 1028 |
+
search();
|
| 1029 |
+
});
|
| 1030 |
+
}
|
| 1031 |
+
|
| 1032 |
+
function populateDockerFacets(f) {
|
| 1033 |
+
const s = state.sel.docker;
|
| 1034 |
+
chipbar($('d-repo-types'), f.repo_types || [], s.repo_types, 6);
|
| 1035 |
+
for (const [key, id, max] of [
|
| 1036 |
+
['accelerators','d-accelerators',8], ['cuda_versions','d-cuda-versions',10],
|
| 1037 |
+
['rocm_versions','d-rocm-versions',8], ['python_versions','d-python-versions',10],
|
| 1038 |
+
['frameworks','d-frameworks',10], ['base_images','d-base-images',10],
|
| 1039 |
+
['operating_systems','d-operating-systems',8],
|
| 1040 |
+
['package_managers','d-package-managers',8], ['services','d-services',10],
|
| 1041 |
+
['node_versions','d-node-versions',8],
|
| 1042 |
+
]) chipbar($(id), f[key] || [], s[key], max);
|
| 1043 |
+
}
|
| 1044 |
+
|
| 1045 |
+
const FACET_POPULATORS = {
|
| 1046 |
+
models: populateModelFacets, kernels: populateKernelFacets,
|
| 1047 |
+
docker: populateDockerFacets,
|
| 1048 |
+
};
|
| 1049 |
+
|
| 1050 |
+
function applyCopy() {
|
| 1051 |
+
const copy = COPY[state.corpus];
|
| 1052 |
+
const facets = state.facets[state.corpus];
|
| 1053 |
+
const total = facets ? facets.total.toLocaleString() : 'β¦';
|
| 1054 |
+
document.title = copy.title;
|
| 1055 |
+
document.querySelector('meta[name="description"]').content = copy.description;
|
| 1056 |
+
$('tagline').textContent = copy.tagline;
|
| 1057 |
+
$('hero-h').innerHTML = copy.hero;
|
| 1058 |
+
$('q').placeholder = copy.placeholder;
|
| 1059 |
+
$('promise').innerHTML = copy.promise(total);
|
| 1060 |
+
$('manifesto').innerHTML = copy.manifesto;
|
| 1061 |
+
$('empty-title').textContent = copy.emptyTitle;
|
| 1062 |
+
$('empty-copy').textContent = copy.emptyCopy;
|
| 1063 |
+
$('footer-main').innerHTML = copy.footer;
|
| 1064 |
+
$('footer-next').textContent = copy.next;
|
| 1065 |
+
$('catalog-copy').textContent = copy.catalog;
|
| 1066 |
+
$('head-count').textContent = facets
|
| 1067 |
+
? `${total} ${HEAD_NOUNS[state.corpus]}` : 'β¦';
|
| 1068 |
+
$('crate-title').textContent = CRATE_TITLES[state.corpus];
|
| 1069 |
+
}
|
| 1070 |
+
|
| 1071 |
+
async function loadFacets(corpus) {
|
| 1072 |
+
if (state.ready[corpus]) return state.facets[corpus];
|
| 1073 |
+
if (state.loading[corpus]) return state.loading[corpus];
|
| 1074 |
+
state.loading[corpus] = getJSON(endpoint('facets', corpus)).then(f => {
|
| 1075 |
+
state.facets[corpus] = f;
|
| 1076 |
+
state.ready[corpus] = true;
|
| 1077 |
+
FACET_POPULATORS[corpus](f);
|
| 1078 |
+
return f;
|
| 1079 |
+
}).finally(() => { state.loading[corpus] = null; });
|
| 1080 |
+
return state.loading[corpus];
|
| 1081 |
+
}
|
| 1082 |
+
|
| 1083 |
+
function resetResults() {
|
| 1084 |
+
state.page = 0;
|
| 1085 |
+
state.total = 0;
|
| 1086 |
+
state.busy = false;
|
| 1087 |
+
$('grid').innerHTML = '';
|
| 1088 |
+
$('crates-head').style.display = 'none';
|
| 1089 |
+
$('empty').style.display = 'none';
|
| 1090 |
+
$('more').style.display = 'none';
|
| 1091 |
+
$('results-wrap').parentElement.classList.remove('loading');
|
| 1092 |
+
}
|
| 1093 |
+
|
| 1094 |
+
async function switchCorpus(corpus) {
|
| 1095 |
+
state.corpus = corpus;
|
| 1096 |
+
state.epoch++;
|
| 1097 |
+
resetResults();
|
| 1098 |
+
$('model-filters').style.display = corpus === 'models' ? '' : 'none';
|
| 1099 |
+
$('kernel-filters').style.display = corpus === 'kernels' ? '' : 'none';
|
| 1100 |
+
$('docker-filters').style.display = corpus === 'docker' ? '' : 'none';
|
| 1101 |
+
$('spread').style.display = 'none';
|
| 1102 |
+
applyCopy();
|
| 1103 |
+
$('status').textContent = WARMING[corpus];
|
| 1104 |
+
try {
|
| 1105 |
+
await loadFacets(corpus);
|
| 1106 |
+
if (state.corpus !== corpus) return;
|
| 1107 |
+
applyCopy();
|
| 1108 |
+
idle();
|
| 1109 |
+
} catch (e) {
|
| 1110 |
+
if (state.corpus === corpus)
|
| 1111 |
+
$('status').innerHTML = `β ${esc(e.message)}`;
|
| 1112 |
+
}
|
| 1113 |
+
}
|
| 1114 |
+
|
| 1115 |
+
function clearModels() {
|
| 1116 |
+
for (const id of ['f-month-from','f-month-to','f-relation','f-license']) $(id).value = '';
|
| 1117 |
+
for (const id of ['f-base','f-pmin','f-pmax','f-ctx','f-config']) $(id).value = '';
|
| 1118 |
+
const s = state.sel.models;
|
| 1119 |
+
s.archs.clear(); s.methods.clear(); s.imports.clear();
|
| 1120 |
+
s.moe = s.custom = false;
|
| 1121 |
+
$('f-moe').classList.remove('on'); $('f-custom').classList.remove('on');
|
| 1122 |
+
$('model-filters').querySelectorAll('.chip.on').forEach(c => c.classList.remove('on'));
|
| 1123 |
+
// clearing restores the product defaults; it does not switch them off, which
|
| 1124 |
+
// would silently widen every search that follows.
|
| 1125 |
+
s.excludedFamilies = new Set(DEFAULT_EXCLUDED_FAMILIES);
|
| 1126 |
+
s.excludeQuantizations = true;
|
| 1127 |
+
$('size-warn').style.display = 'none';
|
| 1128 |
+
renderExclusions();
|
| 1129 |
+
}
|
| 1130 |
+
|
| 1131 |
+
function clearKernels() {
|
| 1132 |
+
for (const id of ['k-month-from','k-month-to','k-author']) $(id).value = '';
|
| 1133 |
+
for (const id of ['k-downloads-min','k-downloads-max','k-variant-min']) $(id).value = '';
|
| 1134 |
+
const s = state.sel.kernels;
|
| 1135 |
+
for (const key of ['languages','accelerators','cuda_archs','dtypes','intrinsics','torch_ops','torch_versions'])
|
| 1136 |
+
s[key].clear();
|
| 1137 |
+
s.has_build_dir = false;
|
| 1138 |
+
$('k-build').classList.remove('on');
|
| 1139 |
+
$('kernel-filters').querySelectorAll('.chip.on').forEach(c => c.classList.remove('on'));
|
| 1140 |
+
}
|
| 1141 |
+
|
| 1142 |
+
function clearDocker() {
|
| 1143 |
+
const s = state.sel.docker;
|
| 1144 |
+
s.repo_types.clear();
|
| 1145 |
+
for (const key of DOCKER_ARRAY_FILTERS) s[key].clear();
|
| 1146 |
+
for (const key of DOCKER_BOOL_FILTERS) s[key] = false;
|
| 1147 |
+
s.includeArchives = false; // back to recipes only
|
| 1148 |
+
for (const id of ['d-compose','d-devcontainer','d-multistage','d-archives'])
|
| 1149 |
+
$(id).classList.remove('on');
|
| 1150 |
+
$('docker-filters').querySelectorAll('.chip.on').forEach(c => c.classList.remove('on'));
|
| 1151 |
+
}
|
| 1152 |
+
|
| 1153 |
+
const CLEARERS = {models: clearModels, kernels: clearKernels, docker: clearDocker};
|
| 1154 |
+
|
| 1155 |
+
async function init() {
|
| 1156 |
+
applyCopy();
|
| 1157 |
+
renderExclusions();
|
| 1158 |
+
$('status').textContent = WARMING.models;
|
| 1159 |
+
try {
|
| 1160 |
+
await loadFacets('models');
|
| 1161 |
+
applyCopy();
|
| 1162 |
+
await loadSpread();
|
| 1163 |
+
idle();
|
| 1164 |
+
} catch (e) {
|
| 1165 |
+
$('status').innerHTML = `β ${esc(e.message)}`;
|
| 1166 |
+
}
|
| 1167 |
+
}
|
| 1168 |
+
|
| 1169 |
+
/* entrance: quiet, three beats */
|
| 1170 |
+
if (window.gsap) {
|
| 1171 |
+
gsap.from('#hdr', { y: -14, opacity: 0, duration: .5, ease: 'power2.out' });
|
| 1172 |
+
gsap.from('#hero-h', { y: 20, opacity: 0, duration: .6, delay: .1, ease: 'power3.out' });
|
| 1173 |
+
gsap.from('#rule i', { scaleX: 0, duration: .4, stagger: .06, delay: .35, ease: 'power2.out' });
|
| 1174 |
+
gsap.from('#term', { y: 14, opacity: 0, duration: .5, delay: .45, ease: 'power2.out' });
|
| 1175 |
+
}
|
| 1176 |
+
|
| 1177 |
+
/* events */
|
| 1178 |
+
$('go').onclick = () => { if ($('q').value.trim()) $('sort').value = 'relevance'; search(); };
|
| 1179 |
+
$('q').addEventListener('keydown', e => { if (e.key === 'Enter') $('go').click(); });
|
| 1180 |
+
$('corpus').onchange = () => switchCorpus($('corpus').value);
|
| 1181 |
+
$('more').onclick = () => { state.page++; search(true); };
|
| 1182 |
+
$('sort').onchange = () => search();
|
| 1183 |
+
for (const id of ['f-month-from','f-month-to','f-relation','f-license'])
|
| 1184 |
+
$(id).onchange = () => search();
|
| 1185 |
+
let deb;
|
| 1186 |
+
for (const id of ['f-base','f-pmin','f-pmax','f-ctx','f-config'])
|
| 1187 |
+
$(id).addEventListener('input', () => { clearTimeout(deb); deb = setTimeout(() => search(), 450); });
|
| 1188 |
+
for (const [id, key] of [['f-moe','moe'],['f-custom','custom']])
|
| 1189 |
+
$(id).onclick = () => { const s = state.sel.models; s[key] = !s[key]; $(id).classList.toggle('on'); search(); };
|
| 1190 |
+
for (const id of ['k-month-from','k-month-to','k-author'])
|
| 1191 |
+
$(id).onchange = () => search();
|
| 1192 |
+
for (const id of ['k-downloads-min','k-downloads-max','k-variant-min'])
|
| 1193 |
+
$(id).addEventListener('input', () => { clearTimeout(deb); deb = setTimeout(() => search(), 450); });
|
| 1194 |
+
$('k-build').onclick = () => {
|
| 1195 |
+
const s = state.sel.kernels;
|
| 1196 |
+
s.has_build_dir = !s.has_build_dir;
|
| 1197 |
+
$('k-build').classList.toggle('on');
|
| 1198 |
+
search();
|
| 1199 |
+
};
|
| 1200 |
+
for (const [id, key] of [['d-compose','has_compose'],['d-devcontainer','has_devcontainer'],
|
| 1201 |
+
['d-multistage','has_multistage'],['d-archives','includeArchives']])
|
| 1202 |
+
$(id).onclick = () => {
|
| 1203 |
+
const s = state.sel.docker;
|
| 1204 |
+
s[key] = !s[key];
|
| 1205 |
+
$(id).classList.toggle('on');
|
| 1206 |
+
search();
|
| 1207 |
+
};
|
| 1208 |
+
$('clear').onclick = () => {
|
| 1209 |
+
$('q').value = ''; $('sort').value = 'newest';
|
| 1210 |
+
CLEARERS[state.corpus]();
|
| 1211 |
+
search();
|
| 1212 |
+
};
|
| 1213 |
+
|
| 1214 |
+
init();
|
| 1215 |
+
</script>
|
| 1216 |
+
</body>
|
| 1217 |
+
</html>
|
tests/.DS_Store
ADDED
|
Binary file (6.15 kB). View file
|
|
|
tests/test_config.py
ADDED
|
@@ -0,0 +1,127 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from pathlib import Path
|
| 2 |
+
|
| 3 |
+
import pytest
|
| 4 |
+
|
| 5 |
+
from bsides.config import Settings
|
| 6 |
+
|
| 7 |
+
|
| 8 |
+
def test_defaults_match_the_production_crawl_contract() -> None:
|
| 9 |
+
settings = Settings.from_env(
|
| 10 |
+
{"HF_TOKEN": "present", "OPENAI_API_KEY": "present"}
|
| 11 |
+
)
|
| 12 |
+
|
| 13 |
+
assert settings.pipelines == (
|
| 14 |
+
"text-generation",
|
| 15 |
+
"text2text-generation",
|
| 16 |
+
)
|
| 17 |
+
assert settings.hub_requests_per_window == 2_400
|
| 18 |
+
assert settings.hub_window_seconds == 300.0
|
| 19 |
+
assert settings.embedding_model == "text-embedding-3-small"
|
| 20 |
+
assert settings.embedding_dimensions == 256
|
| 21 |
+
assert settings.embedding_tokens_per_minute == 4_500_000
|
| 22 |
+
assert settings.chunk_target_tokens == 1_200
|
| 23 |
+
assert settings.chunk_overlap_tokens == 150
|
| 24 |
+
assert settings.max_file_bytes == 2 * 1024 * 1024
|
| 25 |
+
assert settings.max_repo_bytes == 16 * 1024 * 1024
|
| 26 |
+
assert settings.checkpoint_every_repos == 2_500
|
| 27 |
+
assert settings.fetch_queue_multiplier == 4
|
| 28 |
+
assert settings.embed_workers == 8
|
| 29 |
+
assert settings.checkpoint_interval_seconds == 1_800.0
|
| 30 |
+
assert settings.progress_interval_seconds == 10.0
|
| 31 |
+
assert settings.tail_interval_seconds == 900.0
|
| 32 |
+
assert settings.tail_overlap_ids == 100
|
| 33 |
+
assert settings.dataset_repo_id == "juiceb0xc0de/b-sides-v2-index"
|
| 34 |
+
assert settings.run_id == "v2"
|
| 35 |
+
assert settings.lease_ttl_seconds == 1_800.0
|
| 36 |
+
assert settings.state_dir == Path("/tmp/bsides")
|
| 37 |
+
assert settings.mode == "both"
|
| 38 |
+
|
| 39 |
+
|
| 40 |
+
def test_crawl_end_is_an_explicit_inclusive_upper_boundary() -> None:
|
| 41 |
+
settings = Settings.from_env(
|
| 42 |
+
{"BSIDES_CRAWL_END": "2026-08-30T06:00:00Z"}
|
| 43 |
+
)
|
| 44 |
+
|
| 45 |
+
assert settings.crawl_end == "2026-08-30T06:00:00Z"
|
| 46 |
+
|
| 47 |
+
|
| 48 |
+
def test_environment_overrides_are_parsed_without_storing_secret_values() -> None:
|
| 49 |
+
settings = Settings.from_env(
|
| 50 |
+
{
|
| 51 |
+
"HF_TOKEN": "hf-secret",
|
| 52 |
+
"OPENAI_API_KEY": "openai-secret",
|
| 53 |
+
"BSIDES_MODE": "collect",
|
| 54 |
+
"BSIDES_HUB_REQUESTS_PER_5M": "1234",
|
| 55 |
+
"BSIDES_CHECKPOINT_EVERY_REPOS": "17",
|
| 56 |
+
"BSIDES_FETCH_QUEUE_MULTIPLIER": "3",
|
| 57 |
+
"BSIDES_EMBED_WORKERS": "5",
|
| 58 |
+
"BSIDES_EMBEDDING_TPM": "4000000",
|
| 59 |
+
"BSIDES_CHECKPOINT_INTERVAL_SECONDS": "45",
|
| 60 |
+
"BSIDES_PROGRESS_INTERVAL_SECONDS": "7",
|
| 61 |
+
"BSIDES_TAIL_INTERVAL_SECONDS": "60",
|
| 62 |
+
"BSIDES_STATE_DIR": "/tmp/custom-b-sides",
|
| 63 |
+
"BSIDES_DATASET": "owner/index",
|
| 64 |
+
"BSIDES_YEAR": "2025",
|
| 65 |
+
"BSIDES_CRAWL_END": "2026-08-30T06:00:00Z",
|
| 66 |
+
}
|
| 67 |
+
)
|
| 68 |
+
|
| 69 |
+
assert settings.mode == "collect"
|
| 70 |
+
assert settings.hub_requests_per_window == 1_234
|
| 71 |
+
assert settings.checkpoint_every_repos == 17
|
| 72 |
+
assert settings.fetch_queue_multiplier == 3
|
| 73 |
+
assert settings.embed_workers == 5
|
| 74 |
+
assert settings.embedding_tokens_per_minute == 4_000_000
|
| 75 |
+
assert settings.checkpoint_interval_seconds == 45.0
|
| 76 |
+
assert settings.progress_interval_seconds == 7.0
|
| 77 |
+
assert settings.tail_interval_seconds == 60.0
|
| 78 |
+
assert settings.state_dir == Path("/tmp/custom-b-sides")
|
| 79 |
+
assert settings.dataset_repo_id == "owner/index"
|
| 80 |
+
assert settings.crawl_year == 2025
|
| 81 |
+
assert settings.crawl_end == "2026-08-30T06:00:00Z"
|
| 82 |
+
assert settings.hf_token_present is True
|
| 83 |
+
assert settings.openai_api_key_present is True
|
| 84 |
+
assert "hf-secret" not in repr(settings)
|
| 85 |
+
assert "openai-secret" not in repr(settings)
|
| 86 |
+
|
| 87 |
+
|
| 88 |
+
@pytest.mark.parametrize(
|
| 89 |
+
("env", "message"),
|
| 90 |
+
[
|
| 91 |
+
({"BSIDES_MODE": "wrong"}, "BSIDES_MODE"),
|
| 92 |
+
({"BSIDES_HUB_REQUESTS_PER_5M": "0"}, "Hub request budget"),
|
| 93 |
+
({"BSIDES_HUB_REQUESTS_PER_5M": "3000"}, "Hub request budget"),
|
| 94 |
+
({"BSIDES_EMBEDDING_DIMENSIONS": "0"}, "embedding dimensions"),
|
| 95 |
+
({"BSIDES_CHUNK_OVERLAP_TOKENS": "1200"}, "overlap"),
|
| 96 |
+
({"BSIDES_FETCH_QUEUE_MULTIPLIER": "0"}, "queue multiplier"),
|
| 97 |
+
({"BSIDES_EMBED_WORKERS": "0"}, "embedding workers"),
|
| 98 |
+
({"BSIDES_EMBEDDING_TPM": "0"}, "embedding token rate"),
|
| 99 |
+
({"BSIDES_CHECKPOINT_INTERVAL_SECONDS": "0"}, "checkpoint interval"),
|
| 100 |
+
({"BSIDES_PROGRESS_INTERVAL_SECONDS": "0"}, "progress interval"),
|
| 101 |
+
],
|
| 102 |
+
)
|
| 103 |
+
def test_invalid_settings_are_rejected(env: dict[str, str], message: str) -> None:
|
| 104 |
+
with pytest.raises(ValueError, match=message):
|
| 105 |
+
Settings.from_env(env)
|
| 106 |
+
|
| 107 |
+
|
| 108 |
+
@pytest.mark.parametrize(
|
| 109 |
+
("env", "missing"),
|
| 110 |
+
[
|
| 111 |
+
({"OPENAI_API_KEY": "present"}, "HF_TOKEN"),
|
| 112 |
+
({"HF_TOKEN": "present"}, "OPENAI_API_KEY"),
|
| 113 |
+
],
|
| 114 |
+
)
|
| 115 |
+
def test_collection_validation_names_missing_credential(
|
| 116 |
+
env: dict[str, str], missing: str
|
| 117 |
+
) -> None:
|
| 118 |
+
settings = Settings.from_env(env)
|
| 119 |
+
|
| 120 |
+
with pytest.raises(ValueError, match=missing):
|
| 121 |
+
settings.validate_for_collection()
|
| 122 |
+
|
| 123 |
+
|
| 124 |
+
def test_search_mode_does_not_require_collection_credentials() -> None:
|
| 125 |
+
settings = Settings.from_env({"BSIDES_MODE": "search"})
|
| 126 |
+
|
| 127 |
+
settings.validate_for_collection()
|
tests/test_runtime.py
ADDED
|
@@ -0,0 +1,874 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from __future__ import annotations
|
| 2 |
+
|
| 3 |
+
import os
|
| 4 |
+
import pathlib
|
| 5 |
+
import re
|
| 6 |
+
import sqlite3
|
| 7 |
+
import time
|
| 8 |
+
from dataclasses import replace
|
| 9 |
+
|
| 10 |
+
import numpy as np
|
| 11 |
+
import pytest
|
| 12 |
+
from fastapi.testclient import TestClient
|
| 13 |
+
|
| 14 |
+
|
| 15 |
+
MODEL_DDL = """
|
| 16 |
+
CREATE TABLE models (
|
| 17 |
+
row INTEGER PRIMARY KEY, model_id TEXT NOT NULL UNIQUE, author TEXT,
|
| 18 |
+
month TEXT, created_at TEXT, downloads INTEGER, likes INTEGER, tags TEXT,
|
| 19 |
+
base_model TEXT, relation TEXT, license TEXT, params INTEGER,
|
| 20 |
+
model_type TEXT, architectures TEXT, context_len INTEGER,
|
| 21 |
+
num_experts INTEGER, code_imports TEXT, gated INTEGER, config TEXT
|
| 22 |
+
);
|
| 23 |
+
"""
|
| 24 |
+
|
| 25 |
+
KERNEL_DDL = """
|
| 26 |
+
CREATE TABLE kernels (
|
| 27 |
+
row INTEGER PRIMARY KEY, repo_id TEXT NOT NULL UNIQUE, author TEXT,
|
| 28 |
+
created_at TEXT, month TEXT, last_modified TEXT, downloads INTEGER,
|
| 29 |
+
likes INTEGER, tags TEXT, languages TEXT, accelerators TEXT,
|
| 30 |
+
torch_versions TEXT, cuda_archs TEXT, cpu_archs TEXT,
|
| 31 |
+
operating_systems TEXT, dtypes TEXT, intrinsics TEXT, kernel_names TEXT,
|
| 32 |
+
torch_ops TEXT, build_variants TEXT, variant_count INTEGER,
|
| 33 |
+
has_build_dir INTEGER, source_files INTEGER, document TEXT
|
| 34 |
+
);
|
| 35 |
+
"""
|
| 36 |
+
|
| 37 |
+
CONTAINER_DDL = """
|
| 38 |
+
CREATE TABLE containers (
|
| 39 |
+
row INTEGER PRIMARY KEY, repo_id TEXT NOT NULL, repo_type TEXT NOT NULL,
|
| 40 |
+
artifact_path TEXT NOT NULL, artifact_kind TEXT NOT NULL,
|
| 41 |
+
author TEXT, sha TEXT NOT NULL, created_at TEXT, last_modified TEXT,
|
| 42 |
+
downloads INTEGER, likes INTEGER, tags TEXT, description TEXT,
|
| 43 |
+
artifact_files TEXT, qualification_reasons TEXT, base_images TEXT,
|
| 44 |
+
accelerators TEXT, cuda_versions TEXT, rocm_versions TEXT, python_versions TEXT,
|
| 45 |
+
node_versions TEXT, operating_systems TEXT, ports TEXT, services TEXT,
|
| 46 |
+
package_managers TEXT, frameworks TEXT, entrypoints TEXT, commands TEXT,
|
| 47 |
+
has_compose INTEGER, has_devcontainer INTEGER, has_multistage INTEGER,
|
| 48 |
+
source_files INTEGER, source_bytes INTEGER, document TEXT, document_hash TEXT
|
| 49 |
+
);
|
| 50 |
+
"""
|
| 51 |
+
|
| 52 |
+
FIXTURE_MODELS = [
|
| 53 |
+
(0, "indie/alpha", "indie", "2024-01", "2024-01-10T00:00:00Z", 10, 1,
|
| 54 |
+
'["llama","sft"]', None, "finetune", "mit", None, "llama", None, None,
|
| 55 |
+
None, None, 0),
|
| 56 |
+
(1, "indie/beta", "lab", "2025-02", "2025-02-02T00:00:00Z", 500, 9,
|
| 57 |
+
'["mamba"]', None, None, "apache-2.0", None, "mamba", None, None, None,
|
| 58 |
+
None, 0),
|
| 59 |
+
]
|
| 60 |
+
|
| 61 |
+
FIXTURE_KERNELS = [
|
| 62 |
+
(0, "alice/fused-gemm", "alice", "2025-01-10T00:00:00Z", "2025-01",
|
| 63 |
+
"2025-01-11T00:00:00Z", 100, 3, '["kernels"]', '["cuda","python"]',
|
| 64 |
+
'["nvidia"]', '["2.5"]', '["sm_90"]', '[]', '["linux"]',
|
| 65 |
+
'["bf16"]', '["mma_sync"]', '["fused_gemm"]', '["matmul"]',
|
| 66 |
+
'["torch25-cu124"]', 3, 1, 4, "fused gemm kernel"),
|
| 67 |
+
(1, "bob/paged-attention", "bob", "2024-06-02T00:00:00Z", "2024-06",
|
| 68 |
+
"2024-06-03T00:00:00Z", 20, 8, '["kernels"]', '["triton","python"]',
|
| 69 |
+
'["nvidia"]', '["2.4"]', '["sm_80"]', '[]', '["linux"]',
|
| 70 |
+
'["fp32"]', '["tl.dot"]', '["paged_attention"]', '["attention"]',
|
| 71 |
+
'[]', 1, 0, 2, "paged attention kernel"),
|
| 72 |
+
]
|
| 73 |
+
|
| 74 |
+
FIXTURE_CONTAINERS = [
|
| 75 |
+
(0, "alice/cuda-recipe", "model", "Dockerfile", "recipe", "alice",
|
| 76 |
+
"sha-cuda", "2025-02-01T00:00:00Z", "2025-02-02T00:00:00Z", 300, 9,
|
| 77 |
+
'["docker","cuda"]', "CUDA PyTorch recipe", '["Dockerfile"]',
|
| 78 |
+
'["dockerfile","dependencies"]',
|
| 79 |
+
'["nvidia/cuda:12.8.0-runtime-ubuntu22.04"]', '["nvidia"]', '["12.8"]',
|
| 80 |
+
'[]', '["3.11"]', '[]', '["ubuntu22.04"]', '["7860"]', '["api"]',
|
| 81 |
+
'["pip"]', '["pytorch"]', '["python app.py"]', '["uvicorn app:app"]',
|
| 82 |
+
0, 0, 1, 6, 4096, "python 3.11 cuda 12.8 pytorch recipe", "hash-cuda"),
|
| 83 |
+
(1, "bob/rocm-recipe", "space", "docker/Dockerfile", "recipe", "bob",
|
| 84 |
+
"sha-rocm", "2025-03-01T00:00:00Z", "2025-03-02T00:00:00Z", 120, 4,
|
| 85 |
+
'["docker","rocm"]', "ROCm build", '["docker/Dockerfile"]',
|
| 86 |
+
'["dockerfile"]', '["rocm/pytorch:latest"]', '["amd"]', '[]', '["6.1"]',
|
| 87 |
+
'["3.12"]', '["20"]', '["ubuntu24.04"]', '["7860"]', '[]',
|
| 88 |
+
'["apt","pip"]', '["pytorch"]', '[]', '["python launch.py"]',
|
| 89 |
+
0, 0, 0, 5, 3072, "python 3.12 rocm pytorch recipe", "hash-rocm"),
|
| 90 |
+
(2, "carol/compose-stack", "dataset", "compose/Dockerfile", "recipe",
|
| 91 |
+
"carol", "sha-compose", "2025-04-01T00:00:00Z",
|
| 92 |
+
"2025-04-02T00:00:00Z", 80, 6, '["docker","compose"]',
|
| 93 |
+
"Compose-backed stack",
|
| 94 |
+
'["compose/Dockerfile","compose/docker-compose.yml"]',
|
| 95 |
+
'["compose","devcontainer"]', '["python:3.10-slim"]', '[]', '[]', '[]',
|
| 96 |
+
'["3.10"]', '["18"]', '["debian"]', '["5432","8888"]',
|
| 97 |
+
'["jupyter","postgres"]', '["apt","pip"]', '["fastapi"]',
|
| 98 |
+
'["./boot.sh"]', '["docker compose up"]', 1, 1, 0, 8, 8192,
|
| 99 |
+
"compose jupyter postgres fastapi stack", "hash-compose"),
|
| 100 |
+
(3, "dana/archive-bundle", "dataset", "artifacts/container.tar.gz",
|
| 101 |
+
"archive", "dana", "sha-archive", "2025-05-01T00:00:00Z",
|
| 102 |
+
"2025-05-02T00:00:00Z", 40, 1, '["docker","archive"]',
|
| 103 |
+
"Archived CUDA image", '["artifacts/container.tar.gz"]', '["archive"]',
|
| 104 |
+
'["ubuntu:22.04"]', '["nvidia"]', '["12.8"]', '[]', '["3.11"]', '[]',
|
| 105 |
+
'["ubuntu22.04"]', '["8080"]', '["worker"]', '["pip"]',
|
| 106 |
+
'["pytorch"]', '["./serve.sh"]', '["python worker.py"]',
|
| 107 |
+
0, 0, 0, 2, 2048, "python 3.11 cuda 12.8 archive image",
|
| 108 |
+
"hash-archive"),
|
| 109 |
+
]
|
| 110 |
+
|
| 111 |
+
|
| 112 |
+
def write_kernel_index(path, rows=FIXTURE_KERNELS, dimensions=4):
|
| 113 |
+
path.mkdir(parents=True, exist_ok=True)
|
| 114 |
+
connection = sqlite3.connect(path / "meta.sqlite")
|
| 115 |
+
connection.executescript(KERNEL_DDL)
|
| 116 |
+
connection.executemany(
|
| 117 |
+
f"INSERT INTO kernels VALUES ({','.join('?' * 24)})", rows)
|
| 118 |
+
connection.commit()
|
| 119 |
+
connection.close()
|
| 120 |
+
matrix = np.zeros((len(rows), dimensions), dtype=np.float16)
|
| 121 |
+
for row in range(len(rows)):
|
| 122 |
+
matrix[row, row % dimensions] = 1.0
|
| 123 |
+
np.save(path / "embeddings.f16.npy", matrix)
|
| 124 |
+
|
| 125 |
+
|
| 126 |
+
def write_container_index(path, rows=FIXTURE_CONTAINERS, dimensions=4):
|
| 127 |
+
path.mkdir(parents=True, exist_ok=True)
|
| 128 |
+
connection = sqlite3.connect(path / "meta.sqlite")
|
| 129 |
+
connection.executescript(CONTAINER_DDL)
|
| 130 |
+
connection.executemany(
|
| 131 |
+
f"INSERT INTO containers VALUES ({','.join('?' * 35)})", rows)
|
| 132 |
+
connection.commit()
|
| 133 |
+
connection.close()
|
| 134 |
+
matrix = np.zeros((len(rows), dimensions), dtype=np.float16)
|
| 135 |
+
for row in range(len(rows)):
|
| 136 |
+
matrix[row, row % dimensions] = 1.0
|
| 137 |
+
np.save(path / "embeddings.f16.npy", matrix)
|
| 138 |
+
|
| 139 |
+
|
| 140 |
+
@pytest.fixture
|
| 141 |
+
def http_client(tmp_path):
|
| 142 |
+
"""The real app over a real two-row index, swapped in by registry key.
|
| 143 |
+
|
| 144 |
+
TestClient is built without its context manager on purpose: entering the
|
| 145 |
+
lifespan would start the collector thread, which this test is not about.
|
| 146 |
+
"""
|
| 147 |
+
import app as app_module
|
| 148 |
+
|
| 149 |
+
from bsides.search_indexes import ModelAdapter
|
| 150 |
+
|
| 151 |
+
conn = sqlite3.connect(tmp_path / "meta.sqlite")
|
| 152 |
+
conn.executescript(MODEL_DDL)
|
| 153 |
+
conn.executemany(
|
| 154 |
+
f"INSERT INTO models VALUES ({','.join('?' * 19)})",
|
| 155 |
+
[row + (None,) for row in FIXTURE_MODELS])
|
| 156 |
+
conn.commit()
|
| 157 |
+
conn.close()
|
| 158 |
+
matrix = np.zeros((2, 4), dtype=np.float16)
|
| 159 |
+
matrix[0, 0] = matrix[1, 1] = 1.0
|
| 160 |
+
np.save(tmp_path / "embeddings.f16.npy", matrix)
|
| 161 |
+
|
| 162 |
+
original = app_module.registry.index("models")
|
| 163 |
+
spec = replace(app_module.MODEL_SPEC, local_dir=tmp_path, dimensions=4)
|
| 164 |
+
index = app_module.registry.register(ModelAdapter(spec))
|
| 165 |
+
index.load()
|
| 166 |
+
assert index.ready.is_set(), index.error
|
| 167 |
+
try:
|
| 168 |
+
yield TestClient(app_module.app)
|
| 169 |
+
finally:
|
| 170 |
+
app_module.registry.register(original.adapter)
|
| 171 |
+
|
| 172 |
+
|
| 173 |
+
@pytest.fixture
|
| 174 |
+
def kernel_http_client(http_client, tmp_path):
|
| 175 |
+
import app as app_module
|
| 176 |
+
|
| 177 |
+
from bsides.search_indexes import KernelAdapter
|
| 178 |
+
|
| 179 |
+
kernel_dir = tmp_path / "kernels"
|
| 180 |
+
write_kernel_index(kernel_dir)
|
| 181 |
+
original = app_module.registry.index("kernels")
|
| 182 |
+
spec = replace(app_module.KERNEL_SPEC, local_dir=kernel_dir, dimensions=4)
|
| 183 |
+
index = app_module.registry.register(KernelAdapter(spec))
|
| 184 |
+
index.load()
|
| 185 |
+
assert index.ready.is_set(), index.error
|
| 186 |
+
try:
|
| 187 |
+
yield http_client
|
| 188 |
+
finally:
|
| 189 |
+
app_module.registry.register(original.adapter)
|
| 190 |
+
|
| 191 |
+
|
| 192 |
+
@pytest.fixture
|
| 193 |
+
def docker_http_client(http_client, tmp_path):
|
| 194 |
+
import app as app_module
|
| 195 |
+
|
| 196 |
+
from bsides.search_indexes import ContainerAdapter
|
| 197 |
+
|
| 198 |
+
docker_dir = tmp_path / "docker"
|
| 199 |
+
write_container_index(docker_dir)
|
| 200 |
+
original = app_module.registry.index("docker")
|
| 201 |
+
spec = replace(app_module.DOCKER_SPEC, local_dir=docker_dir, dimensions=4)
|
| 202 |
+
index = app_module.registry.register(ContainerAdapter(spec))
|
| 203 |
+
index.load()
|
| 204 |
+
assert index.ready.is_set(), index.error
|
| 205 |
+
try:
|
| 206 |
+
yield http_client
|
| 207 |
+
finally:
|
| 208 |
+
app_module.registry.register(original.adapter)
|
| 209 |
+
|
| 210 |
+
|
| 211 |
+
@pytest.fixture
|
| 212 |
+
def all_corpora_http_client(kernel_http_client, tmp_path):
|
| 213 |
+
import app as app_module
|
| 214 |
+
|
| 215 |
+
from bsides.search_indexes import ContainerAdapter
|
| 216 |
+
|
| 217 |
+
docker_dir = tmp_path / "docker"
|
| 218 |
+
write_container_index(docker_dir)
|
| 219 |
+
original = app_module.registry.index("docker")
|
| 220 |
+
spec = replace(app_module.DOCKER_SPEC, local_dir=docker_dir, dimensions=4)
|
| 221 |
+
index = app_module.registry.register(ContainerAdapter(spec))
|
| 222 |
+
index.load()
|
| 223 |
+
assert index.ready.is_set(), index.error
|
| 224 |
+
try:
|
| 225 |
+
yield kernel_http_client
|
| 226 |
+
finally:
|
| 227 |
+
app_module.registry.register(original.adapter)
|
| 228 |
+
|
| 229 |
+
|
| 230 |
+
def test_app_registers_kernel_corpus_configuration() -> None:
|
| 231 |
+
import app as app_module
|
| 232 |
+
|
| 233 |
+
spec = app_module.registry.index("kernels").spec
|
| 234 |
+
|
| 235 |
+
assert spec is app_module.KERNEL_SPEC
|
| 236 |
+
assert spec.key == "kernels"
|
| 237 |
+
assert spec.dataset_id == os.environ.get(
|
| 238 |
+
"BSIDES_KERNEL_DATASET", "juiceb0xc0de/b-sides-v2-kernels")
|
| 239 |
+
assert spec.local_dir == app_module.INDEX_DIR / "kernels"
|
| 240 |
+
assert spec.table == "kernels"
|
| 241 |
+
assert spec.id_column == "repo_id"
|
| 242 |
+
assert spec.embed_model == "text-embedding-3-small"
|
| 243 |
+
assert spec.dimensions == 256
|
| 244 |
+
|
| 245 |
+
|
| 246 |
+
def test_app_registers_docker_corpus_configuration() -> None:
|
| 247 |
+
import app as app_module
|
| 248 |
+
|
| 249 |
+
spec = app_module.registry.index("docker").spec
|
| 250 |
+
|
| 251 |
+
assert spec is app_module.DOCKER_SPEC
|
| 252 |
+
assert spec.key == "docker"
|
| 253 |
+
assert spec.dataset_id == os.environ.get(
|
| 254 |
+
"BSIDES_CONTAINER_DATASET", "juiceb0xc0de/b-sides-v2-containers")
|
| 255 |
+
assert spec.local_dir == app_module.INDEX_DIR / "docker"
|
| 256 |
+
assert spec.table == "containers"
|
| 257 |
+
assert spec.id_column == "repo_id"
|
| 258 |
+
assert spec.embed_model == "text-embedding-3-small"
|
| 259 |
+
assert spec.dimensions == 256
|
| 260 |
+
|
| 261 |
+
|
| 262 |
+
def test_kernel_facets_and_search_use_kernel_response_shape(
|
| 263 |
+
kernel_http_client) -> None:
|
| 264 |
+
facets = kernel_http_client.get("/api/facets?corpus=kernels")
|
| 265 |
+
search = kernel_http_client.post(
|
| 266 |
+
"/api/search?corpus=kernels",
|
| 267 |
+
json={"sort": "downloads", "filters": {
|
| 268 |
+
"languages": ["cuda", "triton"],
|
| 269 |
+
"has_build_dir": True,
|
| 270 |
+
"variant_count_min": 2,
|
| 271 |
+
}},
|
| 272 |
+
)
|
| 273 |
+
|
| 274 |
+
assert facets.status_code == 200
|
| 275 |
+
assert facets.json()["total"] == 2
|
| 276 |
+
assert {item["v"]: item["n"]
|
| 277 |
+
for item in facets.json()["languages"]} == {
|
| 278 |
+
"python": 2, "cuda": 1, "triton": 1}
|
| 279 |
+
assert search.status_code == 200
|
| 280 |
+
assert set(search.json()) == {"total", "kernels"}
|
| 281 |
+
assert search.json()["total"] == 1
|
| 282 |
+
assert [item["repo_id"] for item in search.json()["kernels"]] == [
|
| 283 |
+
"alice/fused-gemm"]
|
| 284 |
+
assert search.json()["kernels"][0]["languages"] == ["cuda", "python"]
|
| 285 |
+
|
| 286 |
+
|
| 287 |
+
def test_docker_facets_and_search_use_container_response_shape(
|
| 288 |
+
docker_http_client) -> None:
|
| 289 |
+
facets = docker_http_client.get("/api/facets?corpus=docker")
|
| 290 |
+
search = docker_http_client.post(
|
| 291 |
+
"/api/search?corpus=docker",
|
| 292 |
+
json={"sort": "downloads", "filters": {
|
| 293 |
+
"artifact_kinds": ["recipe"],
|
| 294 |
+
"python_versions": ["3.11"],
|
| 295 |
+
"cuda_versions": ["12.8"],
|
| 296 |
+
"frameworks": ["pytorch"],
|
| 297 |
+
}},
|
| 298 |
+
)
|
| 299 |
+
|
| 300 |
+
assert facets.status_code == 200
|
| 301 |
+
assert facets.json()["total"] == 4
|
| 302 |
+
assert {item["v"]: item["n"]
|
| 303 |
+
for item in facets.json()["artifact_kinds"]} == {
|
| 304 |
+
"recipe": 3, "archive": 1}
|
| 305 |
+
assert search.status_code == 200
|
| 306 |
+
assert set(search.json()) == {"total", "containers"}
|
| 307 |
+
assert search.json()["total"] == 1
|
| 308 |
+
assert [item["repo_id"] for item in search.json()["containers"]] == [
|
| 309 |
+
"alice/cuda-recipe"]
|
| 310 |
+
assert search.json()["containers"][0]["base_images"] == [
|
| 311 |
+
"nvidia/cuda:12.8.0-runtime-ubuntu22.04"]
|
| 312 |
+
|
| 313 |
+
|
| 314 |
+
def test_failed_kernel_load_does_not_affect_model_search(
|
| 315 |
+
http_client, tmp_path) -> None:
|
| 316 |
+
import app as app_module
|
| 317 |
+
|
| 318 |
+
from bsides.search_indexes import KernelAdapter
|
| 319 |
+
|
| 320 |
+
bad_dir = tmp_path / "bad-kernels"
|
| 321 |
+
write_kernel_index(bad_dir, rows=FIXTURE_KERNELS[:1], dimensions=8)
|
| 322 |
+
original = app_module.registry.index("kernels")
|
| 323 |
+
spec = replace(app_module.KERNEL_SPEC, local_dir=bad_dir, dimensions=4)
|
| 324 |
+
failed = app_module.registry.register(KernelAdapter(spec))
|
| 325 |
+
failed.load()
|
| 326 |
+
try:
|
| 327 |
+
kernel_response = http_client.get("/api/facets?corpus=kernels")
|
| 328 |
+
model_response = http_client.post(
|
| 329 |
+
"/api/search", json={"sort": "downloads"})
|
| 330 |
+
|
| 331 |
+
assert failed.error and "dimensions" in failed.error
|
| 332 |
+
assert kernel_response.status_code == 503
|
| 333 |
+
assert model_response.status_code == 200
|
| 334 |
+
assert model_response.json()["total"] == 2
|
| 335 |
+
assert set(model_response.json()) == {"total", "models"}
|
| 336 |
+
finally:
|
| 337 |
+
app_module.registry.register(original.adapter)
|
| 338 |
+
|
| 339 |
+
|
| 340 |
+
def test_failed_docker_load_does_not_affect_model_search(
|
| 341 |
+
http_client, tmp_path) -> None:
|
| 342 |
+
import app as app_module
|
| 343 |
+
|
| 344 |
+
from bsides.search_indexes import ContainerAdapter
|
| 345 |
+
|
| 346 |
+
bad_dir = tmp_path / "bad-docker"
|
| 347 |
+
write_container_index(bad_dir, rows=FIXTURE_CONTAINERS[:1], dimensions=8)
|
| 348 |
+
original = app_module.registry.index("docker")
|
| 349 |
+
spec = replace(app_module.DOCKER_SPEC, local_dir=bad_dir, dimensions=4)
|
| 350 |
+
failed = app_module.registry.register(ContainerAdapter(spec))
|
| 351 |
+
failed.load()
|
| 352 |
+
try:
|
| 353 |
+
docker_response = http_client.get("/api/facets?corpus=docker")
|
| 354 |
+
model_response = http_client.post(
|
| 355 |
+
"/api/search", json={"sort": "downloads"})
|
| 356 |
+
|
| 357 |
+
assert failed.error and "dimensions" in failed.error
|
| 358 |
+
assert docker_response.status_code == 503
|
| 359 |
+
assert model_response.status_code == 200
|
| 360 |
+
assert model_response.json()["total"] == 2
|
| 361 |
+
assert set(model_response.json()) == {"total", "models"}
|
| 362 |
+
finally:
|
| 363 |
+
app_module.registry.register(original.adapter)
|
| 364 |
+
|
| 365 |
+
|
| 366 |
+
def test_reload_endpoint_hot_swaps_only_the_kernel_corpus(
|
| 367 |
+
kernel_http_client, monkeypatch) -> None:
|
| 368 |
+
import app as app_module
|
| 369 |
+
|
| 370 |
+
index = app_module.registry.index("kernels")
|
| 371 |
+
target = index.spec.local_dir
|
| 372 |
+
replacement = [(0, *FIXTURE_KERNELS[1][1:])]
|
| 373 |
+
|
| 374 |
+
def downloader(dataset_id, **kwargs):
|
| 375 |
+
assert dataset_id == index.spec.dataset_id
|
| 376 |
+
assert kwargs["local_dir"] == target
|
| 377 |
+
(target / "meta.sqlite").unlink(missing_ok=True)
|
| 378 |
+
(target / "embeddings.f16.npy").unlink(missing_ok=True)
|
| 379 |
+
write_kernel_index(target, rows=replacement, dimensions=4)
|
| 380 |
+
|
| 381 |
+
monkeypatch.setattr(app_module, "RELOAD_TOKEN", "correct-horse")
|
| 382 |
+
monkeypatch.setattr(app_module.registry, "downloader", downloader)
|
| 383 |
+
|
| 384 |
+
response = kernel_http_client.post(
|
| 385 |
+
"/api/reload?corpus=kernels", json={"token": "correct-horse"})
|
| 386 |
+
|
| 387 |
+
assert response.status_code == 200
|
| 388 |
+
assert response.json() == {
|
| 389 |
+
"status": "reload started", "corpus": "kernels"}
|
| 390 |
+
assert _await_rows(index, 1), index.error
|
| 391 |
+
kernel_body = kernel_http_client.post(
|
| 392 |
+
"/api/search?corpus=kernels", json={"sort": "downloads"}).json()
|
| 393 |
+
model_body = kernel_http_client.post(
|
| 394 |
+
"/api/search", json={"sort": "downloads"}).json()
|
| 395 |
+
assert [item["repo_id"] for item in kernel_body["kernels"]] == [
|
| 396 |
+
"bob/paged-attention"]
|
| 397 |
+
assert model_body["total"] == 2
|
| 398 |
+
assert set(model_body) == {"total", "models"}
|
| 399 |
+
|
| 400 |
+
|
| 401 |
+
def test_reload_endpoint_hot_swaps_only_the_docker_corpus(
|
| 402 |
+
docker_http_client, monkeypatch) -> None:
|
| 403 |
+
import app as app_module
|
| 404 |
+
|
| 405 |
+
index = app_module.registry.index("docker")
|
| 406 |
+
target = index.spec.local_dir
|
| 407 |
+
replacement = [(0, *FIXTURE_CONTAINERS[1][1:])]
|
| 408 |
+
|
| 409 |
+
def downloader(dataset_id, **kwargs):
|
| 410 |
+
assert dataset_id == index.spec.dataset_id
|
| 411 |
+
assert kwargs["local_dir"] == target
|
| 412 |
+
(target / "meta.sqlite").unlink(missing_ok=True)
|
| 413 |
+
(target / "embeddings.f16.npy").unlink(missing_ok=True)
|
| 414 |
+
write_container_index(target, rows=replacement, dimensions=4)
|
| 415 |
+
|
| 416 |
+
monkeypatch.setattr(app_module, "RELOAD_TOKEN", "correct-horse")
|
| 417 |
+
monkeypatch.setattr(app_module.registry, "downloader", downloader)
|
| 418 |
+
|
| 419 |
+
response = docker_http_client.post(
|
| 420 |
+
"/api/reload?corpus=docker", json={"token": "correct-horse"})
|
| 421 |
+
|
| 422 |
+
assert response.status_code == 200
|
| 423 |
+
assert response.json() == {
|
| 424 |
+
"status": "reload started", "corpus": "docker"}
|
| 425 |
+
assert _await_rows(index, 1), index.error
|
| 426 |
+
docker_body = docker_http_client.post(
|
| 427 |
+
"/api/search?corpus=docker", json={"sort": "downloads"}).json()
|
| 428 |
+
model_body = docker_http_client.post(
|
| 429 |
+
"/api/search", json={"sort": "downloads"}).json()
|
| 430 |
+
assert [item["repo_id"] for item in docker_body["containers"]] == [
|
| 431 |
+
"bob/rocm-recipe"]
|
| 432 |
+
assert model_body["total"] == 2
|
| 433 |
+
assert set(model_body) == {"total", "models"}
|
| 434 |
+
|
| 435 |
+
|
| 436 |
+
def test_search_response_keeps_the_total_models_envelope(http_client) -> None:
|
| 437 |
+
response = http_client.post("/api/search", json={"sort": "downloads"})
|
| 438 |
+
|
| 439 |
+
assert response.status_code == 200
|
| 440 |
+
body = response.json()
|
| 441 |
+
assert set(body) == {"total", "models"}
|
| 442 |
+
assert body["total"] == 2
|
| 443 |
+
assert [m["model_id"] for m in body["models"]] == [
|
| 444 |
+
"indie/beta", "indie/alpha"]
|
| 445 |
+
assert body["models"][0]["tags"] == ["mamba"] # JSON columns decoded
|
| 446 |
+
|
| 447 |
+
|
| 448 |
+
def test_model_search_exclusions_are_explicit_and_api_compatible(
|
| 449 |
+
http_client) -> None:
|
| 450 |
+
default = http_client.post(
|
| 451 |
+
"/api/search", json={"sort": "downloads"}).json()
|
| 452 |
+
excluded = http_client.post(
|
| 453 |
+
"/api/search",
|
| 454 |
+
json={"sort": "downloads", "exclude_families": ["llama"]},
|
| 455 |
+
)
|
| 456 |
+
|
| 457 |
+
assert excluded.status_code == 200
|
| 458 |
+
assert set(excluded.json()) == {"total", "models"}
|
| 459 |
+
assert {m["model_id"] for m in default["models"]} == {
|
| 460 |
+
"indie/alpha", "indie/beta"}
|
| 461 |
+
assert [m["model_id"] for m in excluded.json()["models"]] == [
|
| 462 |
+
"indie/beta"]
|
| 463 |
+
|
| 464 |
+
|
| 465 |
+
def test_search_defaults_to_the_models_corpus(http_client) -> None:
|
| 466 |
+
default = http_client.post("/api/search", json={"sort": "downloads"})
|
| 467 |
+
explicit = http_client.post(
|
| 468 |
+
"/api/search?corpus=models", json={"sort": "downloads"})
|
| 469 |
+
|
| 470 |
+
assert default.json() == explicit.json()
|
| 471 |
+
|
| 472 |
+
|
| 473 |
+
class RecordingClient:
|
| 474 |
+
"""Embedding client double that records the app's outbound request."""
|
| 475 |
+
|
| 476 |
+
def __init__(self) -> None:
|
| 477 |
+
self.calls: list[dict] = []
|
| 478 |
+
self.embeddings = self
|
| 479 |
+
|
| 480 |
+
def create(self, **kwargs):
|
| 481 |
+
self.calls.append(kwargs)
|
| 482 |
+
return type("Response", (), {
|
| 483 |
+
"data": [type("Item", (), {"embedding": [1.0, 0.0, 0.0, 0.0]})()]
|
| 484 |
+
})()
|
| 485 |
+
|
| 486 |
+
|
| 487 |
+
def test_semantic_searches_use_their_assigned_corpus_clients(
|
| 488 |
+
all_corpora_http_client, monkeypatch) -> None:
|
| 489 |
+
import app as app_module
|
| 490 |
+
|
| 491 |
+
model = RecordingClient()
|
| 492 |
+
kernel = RecordingClient()
|
| 493 |
+
docker = RecordingClient()
|
| 494 |
+
monkeypatch.setattr(app_module, "query_clients", {
|
| 495 |
+
"models": model, "kernels": kernel, "docker": docker,
|
| 496 |
+
})
|
| 497 |
+
|
| 498 |
+
model_response = all_corpora_http_client.post(
|
| 499 |
+
"/api/search", json={"query": "mamba hybrid architecture"})
|
| 500 |
+
kernel_response = all_corpora_http_client.post(
|
| 501 |
+
"/api/search?corpus=kernels", json={"query": "fused attention"})
|
| 502 |
+
docker_response = all_corpora_http_client.post(
|
| 503 |
+
"/api/search?corpus=docker", json={"query": "python 3.11 cuda 12.8"})
|
| 504 |
+
|
| 505 |
+
assert model_response.status_code == 200
|
| 506 |
+
assert kernel_response.status_code == 200
|
| 507 |
+
assert docker_response.status_code == 200
|
| 508 |
+
assert model.calls == [{
|
| 509 |
+
"model": "text-embedding-3-small",
|
| 510 |
+
"dimensions": 4,
|
| 511 |
+
"input": "mamba hybrid architecture",
|
| 512 |
+
}]
|
| 513 |
+
assert kernel.calls == [{
|
| 514 |
+
"model": "text-embedding-3-small",
|
| 515 |
+
"dimensions": 4,
|
| 516 |
+
"input": "fused attention",
|
| 517 |
+
}]
|
| 518 |
+
assert docker.calls == [{
|
| 519 |
+
"model": "text-embedding-3-small",
|
| 520 |
+
"dimensions": 4,
|
| 521 |
+
"input": "python 3.11 cuda 12.8",
|
| 522 |
+
}]
|
| 523 |
+
|
| 524 |
+
|
| 525 |
+
def test_models_client_remains_the_oai_compatibility_shim() -> None:
|
| 526 |
+
import app as app_module
|
| 527 |
+
|
| 528 |
+
assert set(app_module.query_clients) == {"models", "kernels", "docker"}
|
| 529 |
+
assert app_module.oai is app_module.query_clients["models"]
|
| 530 |
+
|
| 531 |
+
|
| 532 |
+
def test_semantic_search_without_a_key_is_unavailable_not_a_crash(
|
| 533 |
+
http_client, monkeypatch) -> None:
|
| 534 |
+
import app as app_module
|
| 535 |
+
|
| 536 |
+
monkeypatch.setitem(app_module.query_clients, "models", None)
|
| 537 |
+
response = http_client.post(
|
| 538 |
+
"/api/search", json={"query": "mamba hybrid architecture"})
|
| 539 |
+
|
| 540 |
+
assert response.status_code == 503
|
| 541 |
+
assert "OPENAI_API_KEY" in response.json()["detail"]
|
| 542 |
+
|
| 543 |
+
|
| 544 |
+
def test_docker_semantic_search_without_a_key_is_unavailable_not_a_crash(
|
| 545 |
+
docker_http_client, monkeypatch) -> None:
|
| 546 |
+
import app as app_module
|
| 547 |
+
|
| 548 |
+
monkeypatch.setitem(app_module.query_clients, "docker", None)
|
| 549 |
+
response = docker_http_client.post(
|
| 550 |
+
"/api/search?corpus=docker", json={"query": "python 3.11 cuda 12.8"})
|
| 551 |
+
model_response = docker_http_client.post(
|
| 552 |
+
"/api/search", json={"sort": "downloads"})
|
| 553 |
+
|
| 554 |
+
assert response.status_code == 503
|
| 555 |
+
assert "OPENAI_DOCKER_API_KEY" in response.json()["detail"]
|
| 556 |
+
assert model_response.status_code == 200
|
| 557 |
+
|
| 558 |
+
|
| 559 |
+
def test_facets_and_picks_still_answer(http_client) -> None:
|
| 560 |
+
facets = http_client.get("/api/facets")
|
| 561 |
+
assert facets.status_code == 200
|
| 562 |
+
assert facets.json()["total"] == 2
|
| 563 |
+
assert {a["v"] for a in facets.json()["archs"]} == {"llama", "mamba"}
|
| 564 |
+
|
| 565 |
+
picks = http_client.get("/api/picks")
|
| 566 |
+
assert picks.status_code == 200
|
| 567 |
+
assert "picks" in picks.json()
|
| 568 |
+
|
| 569 |
+
|
| 570 |
+
def test_unknown_corpus_is_a_404(http_client) -> None:
|
| 571 |
+
assert http_client.get("/api/facets?corpus=bogus").status_code == 404
|
| 572 |
+
|
| 573 |
+
|
| 574 |
+
# ββ reload: the publish_loop.py wire contract βββββββββββββββββββββββββββββββββ
|
| 575 |
+
# publish_loop.reload_space POSTs {"token": ...} and reads 200 / 403 / 503.
|
| 576 |
+
|
| 577 |
+
def _await_rows(index, expected, timeout=10.0):
|
| 578 |
+
deadline = time.monotonic() + timeout
|
| 579 |
+
while time.monotonic() < deadline:
|
| 580 |
+
if index.rows == expected and index.ready.is_set():
|
| 581 |
+
return True
|
| 582 |
+
time.sleep(0.01)
|
| 583 |
+
return False
|
| 584 |
+
|
| 585 |
+
|
| 586 |
+
def test_reload_without_a_configured_token_is_503(http_client, monkeypatch):
|
| 587 |
+
import app as app_module
|
| 588 |
+
|
| 589 |
+
monkeypatch.setattr(app_module, "RELOAD_TOKEN", "")
|
| 590 |
+
response = http_client.post("/api/reload", json={"token": "anything"})
|
| 591 |
+
|
| 592 |
+
assert response.status_code == 503
|
| 593 |
+
assert "BSIDES_RELOAD_TOKEN" in response.json()["detail"]
|
| 594 |
+
|
| 595 |
+
|
| 596 |
+
def test_reload_with_a_wrong_token_is_403(http_client, monkeypatch):
|
| 597 |
+
import app as app_module
|
| 598 |
+
|
| 599 |
+
monkeypatch.setattr(app_module, "RELOAD_TOKEN", "correct-horse")
|
| 600 |
+
index = app_module.registry.index("models")
|
| 601 |
+
|
| 602 |
+
response = http_client.post("/api/reload", json={"token": "guess"})
|
| 603 |
+
|
| 604 |
+
assert response.status_code == 403
|
| 605 |
+
assert index.rows == 2 # untouched
|
| 606 |
+
|
| 607 |
+
|
| 608 |
+
def test_reload_with_an_empty_body_is_rejected(http_client, monkeypatch):
|
| 609 |
+
import app as app_module
|
| 610 |
+
|
| 611 |
+
monkeypatch.setattr(app_module, "RELOAD_TOKEN", "correct-horse")
|
| 612 |
+
assert http_client.post("/api/reload", json={}).status_code == 403
|
| 613 |
+
|
| 614 |
+
|
| 615 |
+
def test_reload_with_the_right_token_swaps_the_index(http_client, monkeypatch):
|
| 616 |
+
import app as app_module
|
| 617 |
+
|
| 618 |
+
index = app_module.registry.index("models")
|
| 619 |
+
target = index.spec.local_dir
|
| 620 |
+
|
| 621 |
+
def downloader(dataset_id, **kwargs):
|
| 622 |
+
(target / "meta.sqlite").unlink(missing_ok=True)
|
| 623 |
+
(target / "embeddings.f16.npy").unlink(missing_ok=True)
|
| 624 |
+
conn = sqlite3.connect(target / "meta.sqlite")
|
| 625 |
+
conn.executescript(MODEL_DDL)
|
| 626 |
+
conn.executemany(
|
| 627 |
+
f"INSERT INTO models VALUES ({','.join('?' * 19)})",
|
| 628 |
+
[FIXTURE_MODELS[0] + (None,)])
|
| 629 |
+
conn.commit()
|
| 630 |
+
conn.close()
|
| 631 |
+
np.save(target / "embeddings.f16.npy",
|
| 632 |
+
np.array([[1, 0, 0, 0]], dtype=np.float16))
|
| 633 |
+
|
| 634 |
+
monkeypatch.setattr(app_module, "RELOAD_TOKEN", "correct-horse")
|
| 635 |
+
monkeypatch.setattr(app_module.registry, "downloader", downloader)
|
| 636 |
+
|
| 637 |
+
response = http_client.post("/api/reload", json={"token": "correct-horse"})
|
| 638 |
+
|
| 639 |
+
assert response.status_code == 200
|
| 640 |
+
assert response.json()["status"] == "reload started"
|
| 641 |
+
assert _await_rows(index, 1), index.error
|
| 642 |
+
assert index.error is None
|
| 643 |
+
# the swap is visible over HTTP, not just in memory
|
| 644 |
+
body = http_client.post("/api/search", json={"sort": "downloads"}).json()
|
| 645 |
+
assert body["total"] == 1
|
| 646 |
+
assert [m["model_id"] for m in body["models"]] == ["indie/alpha"]
|
| 647 |
+
|
| 648 |
+
|
| 649 |
+
def test_reload_of_an_unknown_corpus_is_a_404(http_client, monkeypatch):
|
| 650 |
+
import app as app_module
|
| 651 |
+
|
| 652 |
+
monkeypatch.setattr(app_module, "RELOAD_TOKEN", "correct-horse")
|
| 653 |
+
response = http_client.post(
|
| 654 |
+
"/api/reload?corpus=bogus", json={"token": "correct-horse"})
|
| 655 |
+
|
| 656 |
+
assert response.status_code == 404
|
| 657 |
+
assert not app_module._reload_lock.locked() # gate ran before the lock
|
| 658 |
+
|
| 659 |
+
|
| 660 |
+
def test_endpoints_503_until_the_index_is_ready(tmp_path) -> None:
|
| 661 |
+
import app as app_module
|
| 662 |
+
|
| 663 |
+
from bsides.search_indexes import ModelAdapter
|
| 664 |
+
|
| 665 |
+
original = app_module.registry.index("models")
|
| 666 |
+
spec = replace(app_module.MODEL_SPEC, local_dir=tmp_path)
|
| 667 |
+
app_module.registry.register(ModelAdapter(spec)) # registered, never loaded
|
| 668 |
+
try:
|
| 669 |
+
client = TestClient(app_module.app)
|
| 670 |
+
assert client.post("/api/search", json={}).status_code == 503
|
| 671 |
+
assert client.get("/api/facets").status_code == 503
|
| 672 |
+
finally:
|
| 673 |
+
app_module.registry.register(original.adapter)
|
| 674 |
+
|
| 675 |
+
|
| 676 |
+
def test_search_index_download_excludes_crawler_staging() -> None:
|
| 677 |
+
# The index dataset also carries the multi-GB crawler checkpoints under
|
| 678 |
+
# staging/. The models spec must allow-list only the two index files, so a
|
| 679 |
+
# boot download can never drag the staging tree onto the Space.
|
| 680 |
+
import app as app_module
|
| 681 |
+
|
| 682 |
+
from bsides.search_indexes import download_index
|
| 683 |
+
|
| 684 |
+
calls: list[tuple[str, dict]] = []
|
| 685 |
+
|
| 686 |
+
def downloader(dataset_id: str, **kwargs) -> None:
|
| 687 |
+
calls.append((dataset_id, kwargs))
|
| 688 |
+
|
| 689 |
+
spec = app_module.MODEL_SPEC
|
| 690 |
+
download_index(downloader, spec)
|
| 691 |
+
|
| 692 |
+
assert calls == [
|
| 693 |
+
(
|
| 694 |
+
spec.dataset_id,
|
| 695 |
+
{
|
| 696 |
+
"repo_type": "dataset",
|
| 697 |
+
"local_dir": spec.local_dir,
|
| 698 |
+
"allow_patterns": ["embeddings.f16.npy", "meta.sqlite"],
|
| 699 |
+
},
|
| 700 |
+
)
|
| 701 |
+
]
|
| 702 |
+
|
| 703 |
+
|
| 704 |
+
def test_docker_index_download_allows_only_published_search_files() -> None:
|
| 705 |
+
import app as app_module
|
| 706 |
+
|
| 707 |
+
from bsides.search_indexes import download_index
|
| 708 |
+
|
| 709 |
+
calls: list[tuple[str, dict]] = []
|
| 710 |
+
|
| 711 |
+
def downloader(dataset_id: str, **kwargs) -> None:
|
| 712 |
+
calls.append((dataset_id, kwargs))
|
| 713 |
+
|
| 714 |
+
spec = app_module.DOCKER_SPEC
|
| 715 |
+
download_index(downloader, spec)
|
| 716 |
+
|
| 717 |
+
assert calls == [
|
| 718 |
+
(
|
| 719 |
+
spec.dataset_id,
|
| 720 |
+
{
|
| 721 |
+
"repo_type": "dataset",
|
| 722 |
+
"local_dir": spec.local_dir,
|
| 723 |
+
"allow_patterns": ["embeddings.f16.npy", "meta.sqlite"],
|
| 724 |
+
},
|
| 725 |
+
)
|
| 726 |
+
]
|
| 727 |
+
|
| 728 |
+
|
| 729 |
+
def test_search_query_uses_the_loaded_index_dimensions(tmp_path) -> None:
|
| 730 |
+
# Dimensions sent to the embedding API come off the loaded matrix, not the
|
| 731 |
+
# spec, so a corpus published at a different width still queries correctly.
|
| 732 |
+
from bsides.search_indexes import LoadedIndex, embed_query
|
| 733 |
+
|
| 734 |
+
import app as app_module
|
| 735 |
+
|
| 736 |
+
calls: list[dict] = []
|
| 737 |
+
|
| 738 |
+
class Embeddings:
|
| 739 |
+
def create(self, **kwargs):
|
| 740 |
+
calls.append(kwargs)
|
| 741 |
+
item = type("Item", (), {"embedding": [1.0] + [0.0] * 255})()
|
| 742 |
+
return type("Response", (), {"data": [item]})()
|
| 743 |
+
|
| 744 |
+
index = LoadedIndex(replace(app_module.MODEL_SPEC, local_dir=tmp_path))
|
| 745 |
+
index.matrix = np.zeros((1, 256), dtype=np.float32)
|
| 746 |
+
client = type("Client", (), {"embeddings": Embeddings()})()
|
| 747 |
+
|
| 748 |
+
vector = embed_query(index, client, "odd attention")
|
| 749 |
+
|
| 750 |
+
assert vector.shape == (256,)
|
| 751 |
+
assert calls == [{
|
| 752 |
+
"model": "text-embedding-3-small",
|
| 753 |
+
"dimensions": 256,
|
| 754 |
+
"input": "odd attention",
|
| 755 |
+
}]
|
| 756 |
+
|
| 757 |
+
|
| 758 |
+
# ββ three-corpus frontend contract βββββββββββββββββββββββββββββββββββββββββββ
|
| 759 |
+
# The UI is one hand-written file with no JavaScript test runner, so these read
|
| 760 |
+
# the source and pin the product decisions that regress silently: the corpus
|
| 761 |
+
# list, per-corpus state, the visible default exclusions, and the generic URL
|
| 762 |
+
# builder that a models-vs-kernels ternary would quietly break.
|
| 763 |
+
|
| 764 |
+
DEFAULT_EXCLUDED_FAMILIES = (
|
| 765 |
+
'llama', 'qwen', 'gemma', 'mistral', 'deepseek', 'phi', 'granite')
|
| 766 |
+
|
| 767 |
+
|
| 768 |
+
def _page() -> str:
|
| 769 |
+
path = pathlib.Path(__file__).resolve().parents[1] / 'static' / 'index.html'
|
| 770 |
+
return path.read_text(encoding='utf-8')
|
| 771 |
+
|
| 772 |
+
|
| 773 |
+
def _function_source(page: str, name: str) -> str:
|
| 774 |
+
body = page.split(f'function {name}(', 1)[1]
|
| 775 |
+
return body.split('\n}\n', 1)[0]
|
| 776 |
+
|
| 777 |
+
|
| 778 |
+
def test_frontend_offers_all_three_corpora() -> None:
|
| 779 |
+
page = _page()
|
| 780 |
+
|
| 781 |
+
assert '<option value="models" selected>Models</option>' in page
|
| 782 |
+
assert '<option value="kernels">Kernels</option>' in page
|
| 783 |
+
assert '<option value="docker">Docker Images</option>' in page
|
| 784 |
+
|
| 785 |
+
|
| 786 |
+
def test_frontend_tracks_docker_in_every_corpus_state_map() -> None:
|
| 787 |
+
page = _page()
|
| 788 |
+
|
| 789 |
+
assert 'facets: { models: null, kernels: null, docker: null }' in page
|
| 790 |
+
assert 'ready: { models: false, kernels: false, docker: false }' in page
|
| 791 |
+
assert 'loading: { models: null, kernels: null, docker: null }' in page
|
| 792 |
+
assert 'docker: {' in page.split('sel: {', 1)[1]
|
| 793 |
+
|
| 794 |
+
|
| 795 |
+
def test_frontend_checks_model_exclusions_by_default() -> None:
|
| 796 |
+
page = _page()
|
| 797 |
+
|
| 798 |
+
declaration = re.search(
|
| 799 |
+
r'const DEFAULT_EXCLUDED_FAMILIES = \[(.*?)\];', page, re.S)
|
| 800 |
+
assert declaration is not None
|
| 801 |
+
assert re.findall(r"'([a-z]+)'", declaration.group(1)) == list(
|
| 802 |
+
DEFAULT_EXCLUDED_FAMILIES)
|
| 803 |
+
|
| 804 |
+
# Checked means hidden, and both exclusions start on for the browser only.
|
| 805 |
+
assert 'excludedFamilies: new Set(DEFAULT_EXCLUDED_FAMILIES)' in page
|
| 806 |
+
assert 'excludeQuantizations: true' in page
|
| 807 |
+
assert 'exclude_families: [...state.sel.models.excludedFamilies]' in page
|
| 808 |
+
assert ('exclude_quantizations: state.sel.models.excludeQuantizations'
|
| 809 |
+
in page)
|
| 810 |
+
|
| 811 |
+
|
| 812 |
+
def test_frontend_restores_default_exclusions_when_clearing_the_deck() -> None:
|
| 813 |
+
page = _page()
|
| 814 |
+
source = _function_source(page, 'clearModels')
|
| 815 |
+
|
| 816 |
+
# `clear the deck` restores the product defaults; it must not switch the
|
| 817 |
+
# exclusions off, which would silently widen every subsequent search.
|
| 818 |
+
assert 'DEFAULT_EXCLUDED_FAMILIES' in source
|
| 819 |
+
assert 'excludeQuantizations = true' in source
|
| 820 |
+
|
| 821 |
+
|
| 822 |
+
def test_frontend_keeps_model_size_as_precise_billion_boxes() -> None:
|
| 823 |
+
page = _page()
|
| 824 |
+
|
| 825 |
+
assert 'id="f-pmin"' in page and 'id="f-pmax"' in page
|
| 826 |
+
assert page.count('step="0.1"') == 2
|
| 827 |
+
assert 'Model size (billions)' in page
|
| 828 |
+
assert '1.5Bβ3.5B' in page
|
| 829 |
+
assert '* 1e9' in page
|
| 830 |
+
# An inverted range is flagged rather than sent as an impossible request.
|
| 831 |
+
assert 'id="size-warn"' in page
|
| 832 |
+
|
| 833 |
+
|
| 834 |
+
def test_frontend_defaults_docker_to_recipes_with_an_archive_toggle() -> None:
|
| 835 |
+
page = _page()
|
| 836 |
+
source = _function_source(page, 'buildDockerReq')
|
| 837 |
+
|
| 838 |
+
assert 'includeArchives: false' in page
|
| 839 |
+
assert 'id="d-archives"' in page
|
| 840 |
+
assert "filters.artifact_kinds = ['recipe']" in source
|
| 841 |
+
# Enabling archives removes the constraint instead of adding a second kind.
|
| 842 |
+
assert 'if (!s.includeArchives)' in source
|
| 843 |
+
assert "'archive'" not in source
|
| 844 |
+
|
| 845 |
+
|
| 846 |
+
def test_frontend_routes_every_corpus_through_one_url_builder() -> None:
|
| 847 |
+
page = _page()
|
| 848 |
+
|
| 849 |
+
assert ("const resultKey = {models:'models', kernels:'kernels', "
|
| 850 |
+
"docker:'containers'}" in page)
|
| 851 |
+
assert 'const endpoint = (kind, corpus) =>' in page
|
| 852 |
+
# No binary ternary may survive, or Docker silently routes to kernels.
|
| 853 |
+
assert "'/api/search?corpus=kernels'" not in page
|
| 854 |
+
assert "'/api/facets?corpus=kernels'" not in page
|
| 855 |
+
|
| 856 |
+
|
| 857 |
+
def test_frontend_escapes_every_docker_card_value() -> None:
|
| 858 |
+
page = _page()
|
| 859 |
+
source = _function_source(page, 'dockerCard')
|
| 860 |
+
|
| 861 |
+
assert 'esc(' in source
|
| 862 |
+
# No raw record field may reach the template unescaped.
|
| 863 |
+
assert re.search(r'\$\{c\.[A-Za-z_]+\}', source) is None
|
| 864 |
+
|
| 865 |
+
|
| 866 |
+
def test_frontend_links_docker_artifacts_to_valid_hugging_face_targets() -> None:
|
| 867 |
+
page = _page()
|
| 868 |
+
source = _function_source(page, 'containerHref')
|
| 869 |
+
|
| 870 |
+
assert "'spaces/'" in source and "'datasets/'" in source
|
| 871 |
+
assert 'blob/' in source
|
| 872 |
+
# Bucket rows and synthetic <recipe> paths have no blob URL; those fall
|
| 873 |
+
# back to the repository landing page instead of a 404.
|
| 874 |
+
assert '<recipe>' in source
|
tests/test_search_indexes.py
ADDED
|
@@ -0,0 +1,1287 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Tests for the reusable multi-corpus index core (bsides.search_indexes).
|
| 2 |
+
|
| 3 |
+
Covers the loader validation contract, the registry's failure isolation,
|
| 4 |
+
the ModelAdapter (today's model behavior, relocated), and the shared
|
| 5 |
+
ranking functions. Fixtures write real `meta.sqlite` + `embeddings.f16.npy`
|
| 6 |
+
pairs to tmp dirs so the loader is exercised exactly as on the Space.
|
| 7 |
+
"""
|
| 8 |
+
|
| 9 |
+
from __future__ import annotations
|
| 10 |
+
|
| 11 |
+
import json
|
| 12 |
+
import sqlite3
|
| 13 |
+
from pathlib import Path
|
| 14 |
+
from types import SimpleNamespace
|
| 15 |
+
|
| 16 |
+
import numpy as np
|
| 17 |
+
import pytest
|
| 18 |
+
|
| 19 |
+
from bsides.search_indexes import (
|
| 20 |
+
MAX_QUERY_CHARS,
|
| 21 |
+
MATRIX_NAME,
|
| 22 |
+
SQLITE_NAME,
|
| 23 |
+
IndexRegistry,
|
| 24 |
+
IndexSpec,
|
| 25 |
+
LoadedIndex,
|
| 26 |
+
ModelAdapter,
|
| 27 |
+
SearchReq,
|
| 28 |
+
UnavailableError,
|
| 29 |
+
download_index,
|
| 30 |
+
embed_query,
|
| 31 |
+
semantic_rank,
|
| 32 |
+
structured_page,
|
| 33 |
+
)
|
| 34 |
+
|
| 35 |
+
# CARD_COLS + created_at (needed by the sort paths), matching the published
|
| 36 |
+
# models schema subset the app actually reads.
|
| 37 |
+
MODEL_DDL = """
|
| 38 |
+
CREATE TABLE models (
|
| 39 |
+
row INTEGER PRIMARY KEY,
|
| 40 |
+
model_id TEXT NOT NULL UNIQUE,
|
| 41 |
+
author TEXT,
|
| 42 |
+
month TEXT,
|
| 43 |
+
created_at TEXT,
|
| 44 |
+
downloads INTEGER,
|
| 45 |
+
likes INTEGER,
|
| 46 |
+
tags TEXT,
|
| 47 |
+
base_model TEXT,
|
| 48 |
+
relation TEXT,
|
| 49 |
+
license TEXT,
|
| 50 |
+
params INTEGER,
|
| 51 |
+
model_type TEXT,
|
| 52 |
+
architectures TEXT,
|
| 53 |
+
context_len INTEGER,
|
| 54 |
+
num_experts INTEGER,
|
| 55 |
+
code_imports TEXT,
|
| 56 |
+
gated INTEGER,
|
| 57 |
+
config TEXT
|
| 58 |
+
);
|
| 59 |
+
"""
|
| 60 |
+
|
| 61 |
+
KERNEL_DDL = """
|
| 62 |
+
CREATE TABLE kernels (
|
| 63 |
+
row INTEGER PRIMARY KEY, repo_id TEXT NOT NULL UNIQUE, author TEXT,
|
| 64 |
+
created_at TEXT, month TEXT, last_modified TEXT, downloads INTEGER,
|
| 65 |
+
likes INTEGER, tags TEXT, languages TEXT, accelerators TEXT,
|
| 66 |
+
torch_versions TEXT, cuda_archs TEXT, cpu_archs TEXT,
|
| 67 |
+
operating_systems TEXT, dtypes TEXT, intrinsics TEXT, kernel_names TEXT,
|
| 68 |
+
torch_ops TEXT, build_variants TEXT, variant_count INTEGER,
|
| 69 |
+
has_build_dir INTEGER, source_files INTEGER, document TEXT
|
| 70 |
+
);
|
| 71 |
+
"""
|
| 72 |
+
|
| 73 |
+
CONTAINER_DDL = """
|
| 74 |
+
CREATE TABLE containers (
|
| 75 |
+
row INTEGER PRIMARY KEY, repo_id TEXT NOT NULL, repo_type TEXT NOT NULL,
|
| 76 |
+
artifact_path TEXT NOT NULL, artifact_kind TEXT NOT NULL,
|
| 77 |
+
author TEXT, sha TEXT NOT NULL, created_at TEXT, last_modified TEXT,
|
| 78 |
+
downloads INTEGER, likes INTEGER, tags TEXT, description TEXT,
|
| 79 |
+
artifact_files TEXT, qualification_reasons TEXT, base_images TEXT,
|
| 80 |
+
accelerators TEXT, cuda_versions TEXT, rocm_versions TEXT, python_versions TEXT,
|
| 81 |
+
node_versions TEXT, operating_systems TEXT, ports TEXT, services TEXT,
|
| 82 |
+
package_managers TEXT, frameworks TEXT, entrypoints TEXT, commands TEXT,
|
| 83 |
+
has_compose INTEGER, has_devcontainer INTEGER, has_multistage INTEGER,
|
| 84 |
+
source_files INTEGER, source_bytes INTEGER, document TEXT, document_hash TEXT
|
| 85 |
+
);
|
| 86 |
+
"""
|
| 87 |
+
|
| 88 |
+
|
| 89 |
+
def model_row(row, model_id, *, author='indie', month='2024-01',
|
| 90 |
+
created_at='2024-01-15T00:00:00Z', downloads=0, likes=0,
|
| 91 |
+
tags='[]', base_model=None, relation=None, license=None,
|
| 92 |
+
params=None, model_type=None, architectures=None,
|
| 93 |
+
context_len=None, num_experts=None, code_imports=None,
|
| 94 |
+
gated=0, config=None):
|
| 95 |
+
return (row, model_id, author, month, created_at, downloads, likes, tags,
|
| 96 |
+
base_model, relation, license, params, model_type, architectures,
|
| 97 |
+
context_len, num_experts, code_imports, gated, config)
|
| 98 |
+
|
| 99 |
+
|
| 100 |
+
def kernel_row(row, repo_id, *, author='alice', month='2025-01',
|
| 101 |
+
created_at='2025-01-15T00:00:00Z', downloads=100, likes=2,
|
| 102 |
+
tags='["kernels"]', languages='["cuda","python"]',
|
| 103 |
+
accelerators='["nvidia"]', torch_versions='["2.5"]',
|
| 104 |
+
cuda_archs='["sm_90"]', cpu_archs='[]',
|
| 105 |
+
operating_systems='["linux"]', dtypes='["bf16","fp16"]',
|
| 106 |
+
intrinsics='["mma_sync","__syncthreads"]',
|
| 107 |
+
kernel_names='["fused_gemm"]', torch_ops='["matmul"]',
|
| 108 |
+
build_variants='["torch25-cu124"]', variant_count=3,
|
| 109 |
+
has_build_dir=1, source_files=4, document='fused gemm kernel'):
|
| 110 |
+
return (row, repo_id, author, created_at, month, created_at, downloads,
|
| 111 |
+
likes, tags, languages, accelerators, torch_versions, cuda_archs,
|
| 112 |
+
cpu_archs, operating_systems, dtypes, intrinsics, kernel_names,
|
| 113 |
+
torch_ops, build_variants, variant_count, has_build_dir,
|
| 114 |
+
source_files, document)
|
| 115 |
+
|
| 116 |
+
|
| 117 |
+
def _json_text(value):
|
| 118 |
+
return json.dumps(value)
|
| 119 |
+
|
| 120 |
+
|
| 121 |
+
def container_row(
|
| 122 |
+
row, repo_id, *, repo_type='model', artifact_path='Dockerfile',
|
| 123 |
+
artifact_kind='recipe', author='alice', sha='sha-1',
|
| 124 |
+
created_at='2025-01-15T00:00:00Z',
|
| 125 |
+
last_modified='2025-01-16T00:00:00Z', downloads=100, likes=2,
|
| 126 |
+
tags=None, description='container recipe', artifact_files=None,
|
| 127 |
+
qualification_reasons=None, base_images=None, accelerators=None,
|
| 128 |
+
cuda_versions=None, rocm_versions=None, python_versions=None,
|
| 129 |
+
node_versions=None, operating_systems=None, ports=None, services=None,
|
| 130 |
+
package_managers=None, frameworks=None, entrypoints=None,
|
| 131 |
+
commands=None, has_compose=0, has_devcontainer=0, has_multistage=0,
|
| 132 |
+
source_files=4, source_bytes=1024, document='container search doc',
|
| 133 |
+
document_hash='hash-1'):
|
| 134 |
+
return (
|
| 135 |
+
row, repo_id, repo_type, artifact_path, artifact_kind, author, sha,
|
| 136 |
+
created_at, last_modified, downloads, likes,
|
| 137 |
+
_json_text(tags or []), description, _json_text(artifact_files or []),
|
| 138 |
+
_json_text(qualification_reasons or []), _json_text(base_images or []),
|
| 139 |
+
_json_text(accelerators or []), _json_text(cuda_versions or []),
|
| 140 |
+
_json_text(rocm_versions or []), _json_text(python_versions or []),
|
| 141 |
+
_json_text(node_versions or []),
|
| 142 |
+
_json_text(operating_systems or []), _json_text(ports or []),
|
| 143 |
+
_json_text(services or []), _json_text(package_managers or []),
|
| 144 |
+
_json_text(frameworks or []), _json_text(entrypoints or []),
|
| 145 |
+
_json_text(commands or []), has_compose, has_devcontainer,
|
| 146 |
+
has_multistage, source_files, source_bytes, document, document_hash,
|
| 147 |
+
)
|
| 148 |
+
|
| 149 |
+
|
| 150 |
+
def basis(n, dim=4):
|
| 151 |
+
"""n normalized basis vectors (row i -> dimension i % dim)."""
|
| 152 |
+
m = np.zeros((n, dim), dtype=np.float32)
|
| 153 |
+
for i in range(n):
|
| 154 |
+
m[i, i % dim] = 1.0
|
| 155 |
+
return m
|
| 156 |
+
|
| 157 |
+
|
| 158 |
+
def write_model_db(dir_path, rows):
|
| 159 |
+
conn = sqlite3.connect(Path(dir_path) / SQLITE_NAME)
|
| 160 |
+
conn.executescript(MODEL_DDL)
|
| 161 |
+
conn.executemany(
|
| 162 |
+
f"INSERT INTO models VALUES ({','.join('?' * 19)})", rows)
|
| 163 |
+
conn.commit()
|
| 164 |
+
conn.close()
|
| 165 |
+
|
| 166 |
+
|
| 167 |
+
def write_matrix(dir_path, vectors):
|
| 168 |
+
np.save(Path(dir_path) / MATRIX_NAME, vectors.astype(np.float16))
|
| 169 |
+
|
| 170 |
+
|
| 171 |
+
def write_kernel_db(dir_path, rows):
|
| 172 |
+
conn = sqlite3.connect(Path(dir_path) / SQLITE_NAME)
|
| 173 |
+
conn.executescript(KERNEL_DDL)
|
| 174 |
+
conn.executemany(
|
| 175 |
+
f"INSERT INTO kernels VALUES ({','.join('?' * 24)})", rows)
|
| 176 |
+
conn.commit()
|
| 177 |
+
conn.close()
|
| 178 |
+
|
| 179 |
+
|
| 180 |
+
def write_container_db(dir_path, rows):
|
| 181 |
+
conn = sqlite3.connect(Path(dir_path) / SQLITE_NAME)
|
| 182 |
+
conn.executescript(CONTAINER_DDL)
|
| 183 |
+
conn.executemany(
|
| 184 |
+
f"INSERT INTO containers VALUES ({','.join('?' * 35)})", rows)
|
| 185 |
+
conn.commit()
|
| 186 |
+
conn.close()
|
| 187 |
+
|
| 188 |
+
|
| 189 |
+
def write_index_dir(dir_path, rows, vectors):
|
| 190 |
+
write_model_db(dir_path, rows)
|
| 191 |
+
write_matrix(dir_path, vectors)
|
| 192 |
+
|
| 193 |
+
|
| 194 |
+
def make_spec(local_dir, **overrides):
|
| 195 |
+
base = dict(
|
| 196 |
+
key='models',
|
| 197 |
+
dataset_id='owner/corpus',
|
| 198 |
+
local_dir=Path(local_dir),
|
| 199 |
+
table='models',
|
| 200 |
+
id_column='model_id',
|
| 201 |
+
embed_model='text-embedding-3-small',
|
| 202 |
+
dimensions=4,
|
| 203 |
+
)
|
| 204 |
+
base.update(overrides)
|
| 205 |
+
return IndexSpec(**base)
|
| 206 |
+
|
| 207 |
+
|
| 208 |
+
FIXTURE_ROWS = [
|
| 209 |
+
model_row(0, 'indie/alpha', downloads=10, likes=1, tags='["llama","sft"]',
|
| 210 |
+
model_type='llama', license='mit', relation='finetune',
|
| 211 |
+
base_model='meta-llama/Llama-3-8B', month='2024-01',
|
| 212 |
+
created_at='2024-01-10T00:00:00Z', code_imports='["flash_attn"]',
|
| 213 |
+
architectures='["LlamaForCausalLM"]', params=8_000_000_000,
|
| 214 |
+
context_len=8192),
|
| 215 |
+
model_row(1, 'indie/beta', author='lab', downloads=500, likes=9,
|
| 216 |
+
tags='["mamba"]', model_type='mamba', license='apache-2.0',
|
| 217 |
+
month='2025-02', created_at='2025-02-02T00:00:00Z'),
|
| 218 |
+
model_row(2, 'indie/gamma', downloads=50, likes=3, tags='["qwen2","dpo"]',
|
| 219 |
+
model_type='qwen2', month='2024-03',
|
| 220 |
+
created_at='2024-03-05T00:00:00Z', num_experts=8,
|
| 221 |
+
code_imports='["triton"]', gated=1),
|
| 222 |
+
]
|
| 223 |
+
|
| 224 |
+
EXCLUSION_ROWS = [
|
| 225 |
+
model_row(
|
| 226 |
+
0, 'indie/llama-derivative', tags='["research"]',
|
| 227 |
+
base_model='meta-llama/Llama-3', model_type='transformer'),
|
| 228 |
+
model_row(
|
| 229 |
+
1, 'indie/qwen-tagged', tags='["Qwen3"]',
|
| 230 |
+
model_type='transformer'),
|
| 231 |
+
model_row(
|
| 232 |
+
2, 'indie/mistral-architecture',
|
| 233 |
+
architectures='["MistralForCausalLM"]', model_type='transformer'),
|
| 234 |
+
model_row(
|
| 235 |
+
3, 'indie/gptq-repack', tags='["gptq"]',
|
| 236 |
+
config='{"quantization_config": {"bits": 4}}'),
|
| 237 |
+
model_row(
|
| 238 |
+
4, 'indie/native-fp8-research', tags='["fp8"]',
|
| 239 |
+
model_type='research', config='{"torch_dtype": "float8_e4m3fn"}'),
|
| 240 |
+
model_row(
|
| 241 |
+
5, 'indie/native-int8-research', tags='["int8"]',
|
| 242 |
+
model_type='research', config='{"torch_dtype": "int8"}'),
|
| 243 |
+
model_row(
|
| 244 |
+
6, 'indie/config-null-quant', model_type='research',
|
| 245 |
+
config='{"quantization_config": null}'),
|
| 246 |
+
model_row(
|
| 247 |
+
7, 'indie/malformed-config', model_type='research',
|
| 248 |
+
config='not-json'),
|
| 249 |
+
]
|
| 250 |
+
|
| 251 |
+
KERNEL_ROWS = [
|
| 252 |
+
kernel_row(0, 'alice/fused-gemm'),
|
| 253 |
+
kernel_row(
|
| 254 |
+
1, 'bob/paged-attention', author='bob', month='2024-06',
|
| 255 |
+
created_at='2024-06-02T00:00:00Z', downloads=20, likes=8,
|
| 256 |
+
languages='["triton","python"]', torch_versions='["2.4"]',
|
| 257 |
+
cuda_archs='["sm_80"]', dtypes='["fp32"]',
|
| 258 |
+
intrinsics='["tl.dot"]', kernel_names='["paged_attention"]',
|
| 259 |
+
torch_ops='["attention"]', build_variants='[]', variant_count=1,
|
| 260 |
+
has_build_dir=0, source_files=2, document='paged attention kernel'),
|
| 261 |
+
kernel_row(
|
| 262 |
+
2, 'alice/int8-conv', month='2025-02',
|
| 263 |
+
created_at='2025-02-03T00:00:00Z', downloads=500, likes=5,
|
| 264 |
+
languages='["cpp"]', accelerators='["amd"]',
|
| 265 |
+
torch_versions='["2.5"]', cuda_archs='[]', cpu_archs='["x86_64"]',
|
| 266 |
+
dtypes='["int8"]', intrinsics='["avx2"]',
|
| 267 |
+
kernel_names='["int8_conv"]', torch_ops='["conv2d"]',
|
| 268 |
+
build_variants='["rocm"]', variant_count=5, has_build_dir=1,
|
| 269 |
+
source_files=6, document='int8 convolution kernel'),
|
| 270 |
+
]
|
| 271 |
+
|
| 272 |
+
CONTAINER_ROWS = [
|
| 273 |
+
container_row(
|
| 274 |
+
0, 'alice/cuda-recipe', repo_type='model', artifact_path='Dockerfile',
|
| 275 |
+
artifact_kind='recipe', author='alice', sha='sha-cuda',
|
| 276 |
+
created_at='2025-02-01T00:00:00Z',
|
| 277 |
+
last_modified='2025-02-02T00:00:00Z', downloads=300, likes=9,
|
| 278 |
+
tags=['docker', 'cuda'], description='CUDA PyTorch recipe',
|
| 279 |
+
artifact_files=['Dockerfile'],
|
| 280 |
+
qualification_reasons=['dockerfile', 'dependencies'],
|
| 281 |
+
base_images=['nvidia/cuda:12.8.0-runtime-ubuntu22.04'],
|
| 282 |
+
accelerators=['nvidia'], cuda_versions=['12.8'], rocm_versions=[],
|
| 283 |
+
python_versions=['3.11'], node_versions=[],
|
| 284 |
+
operating_systems=['ubuntu22.04'], ports=['7860'], services=['api'],
|
| 285 |
+
package_managers=['pip'], frameworks=['pytorch'],
|
| 286 |
+
entrypoints=['python app.py'], commands=['uvicorn app:app'],
|
| 287 |
+
has_compose=0, has_devcontainer=0, has_multistage=1, source_files=6,
|
| 288 |
+
source_bytes=4096,
|
| 289 |
+
document='python 3.11 cuda 12.8 pytorch recipe',
|
| 290 |
+
document_hash='hash-cuda'),
|
| 291 |
+
container_row(
|
| 292 |
+
1, 'bob/rocm-recipe', repo_type='space',
|
| 293 |
+
artifact_path='docker/Dockerfile', artifact_kind='recipe',
|
| 294 |
+
author='bob', sha='sha-rocm', created_at='2025-03-01T00:00:00Z',
|
| 295 |
+
last_modified='2025-03-02T00:00:00Z', downloads=120, likes=4,
|
| 296 |
+
tags=['docker', 'rocm'], description='ROCm build',
|
| 297 |
+
artifact_files=['docker/Dockerfile'],
|
| 298 |
+
qualification_reasons=['dockerfile'],
|
| 299 |
+
base_images=['rocm/pytorch:latest'], accelerators=['amd'],
|
| 300 |
+
cuda_versions=[], rocm_versions=['6.1'], python_versions=['3.12'],
|
| 301 |
+
node_versions=['20'], operating_systems=['ubuntu24.04'],
|
| 302 |
+
ports=['7860'], services=[], package_managers=['apt', 'pip'],
|
| 303 |
+
frameworks=['pytorch'], entrypoints=[], commands=['python launch.py'],
|
| 304 |
+
has_compose=0, has_devcontainer=0, has_multistage=0, source_files=5,
|
| 305 |
+
source_bytes=3072, document='python 3.12 rocm pytorch recipe',
|
| 306 |
+
document_hash='hash-rocm'),
|
| 307 |
+
container_row(
|
| 308 |
+
2, 'carol/compose-stack', repo_type='dataset',
|
| 309 |
+
artifact_path='compose/Dockerfile', artifact_kind='recipe',
|
| 310 |
+
author='carol', sha='sha-compose', created_at='2025-04-01T00:00:00Z',
|
| 311 |
+
last_modified='2025-04-02T00:00:00Z', downloads=80, likes=6,
|
| 312 |
+
tags=['docker', 'compose'], description='Compose-backed stack',
|
| 313 |
+
artifact_files=['compose/Dockerfile', 'compose/docker-compose.yml'],
|
| 314 |
+
qualification_reasons=['compose', 'devcontainer'],
|
| 315 |
+
base_images=['python:3.10-slim'], accelerators=[],
|
| 316 |
+
cuda_versions=[], rocm_versions=[], python_versions=['3.10'],
|
| 317 |
+
node_versions=['18'], operating_systems=['debian'],
|
| 318 |
+
ports=['5432', '8888'], services=['jupyter', 'postgres'],
|
| 319 |
+
package_managers=['apt', 'pip'], frameworks=['fastapi'],
|
| 320 |
+
entrypoints=['./boot.sh'], commands=['docker compose up'],
|
| 321 |
+
has_compose=1, has_devcontainer=1, has_multistage=0, source_files=8,
|
| 322 |
+
source_bytes=8192, document='compose jupyter postgres fastapi stack',
|
| 323 |
+
document_hash='hash-compose'),
|
| 324 |
+
container_row(
|
| 325 |
+
3, 'dana/archive-bundle', repo_type='dataset',
|
| 326 |
+
artifact_path='artifacts/container.tar.gz', artifact_kind='archive',
|
| 327 |
+
author='dana', sha='sha-archive', created_at='2025-05-01T00:00:00Z',
|
| 328 |
+
last_modified='2025-05-02T00:00:00Z', downloads=40, likes=1,
|
| 329 |
+
tags=['docker', 'archive'], description='Archived CUDA image',
|
| 330 |
+
artifact_files=['artifacts/container.tar.gz'],
|
| 331 |
+
qualification_reasons=['archive'],
|
| 332 |
+
base_images=['ubuntu:22.04'], accelerators=['nvidia'],
|
| 333 |
+
cuda_versions=['12.8'], rocm_versions=[], python_versions=['3.11'],
|
| 334 |
+
node_versions=[], operating_systems=['ubuntu22.04'],
|
| 335 |
+
ports=['8080'], services=['worker'], package_managers=['pip'],
|
| 336 |
+
frameworks=['pytorch'], entrypoints=['./serve.sh'],
|
| 337 |
+
commands=['python worker.py'], has_compose=0, has_devcontainer=0,
|
| 338 |
+
has_multistage=0, source_files=2, source_bytes=2048,
|
| 339 |
+
document='python 3.11 cuda 12.8 archive image',
|
| 340 |
+
document_hash='hash-archive'),
|
| 341 |
+
]
|
| 342 |
+
|
| 343 |
+
|
| 344 |
+
@pytest.fixture
|
| 345 |
+
def loaded_models(tmp_path):
|
| 346 |
+
write_index_dir(tmp_path, FIXTURE_ROWS, basis(3))
|
| 347 |
+
spec = make_spec(tmp_path)
|
| 348 |
+
index = LoadedIndex(spec, ModelAdapter(spec))
|
| 349 |
+
index.load()
|
| 350 |
+
assert index.ready.is_set() # fixture sanity
|
| 351 |
+
return index
|
| 352 |
+
|
| 353 |
+
|
| 354 |
+
@pytest.fixture
|
| 355 |
+
def loaded_exclusion_models(tmp_path):
|
| 356 |
+
write_index_dir(tmp_path, EXCLUSION_ROWS, basis(len(EXCLUSION_ROWS)))
|
| 357 |
+
spec = make_spec(tmp_path)
|
| 358 |
+
index = LoadedIndex(spec, ModelAdapter(spec))
|
| 359 |
+
index.load()
|
| 360 |
+
assert index.ready.is_set(), index.error
|
| 361 |
+
return index
|
| 362 |
+
|
| 363 |
+
|
| 364 |
+
def load_kernels(tmp_path):
|
| 365 |
+
from bsides.search_indexes import KernelAdapter
|
| 366 |
+
|
| 367 |
+
write_kernel_db(tmp_path, KERNEL_ROWS)
|
| 368 |
+
write_matrix(tmp_path, basis(3))
|
| 369 |
+
spec = make_spec(
|
| 370 |
+
tmp_path, key='kernels', dataset_id='owner/kernels', table='kernels',
|
| 371 |
+
id_column='repo_id')
|
| 372 |
+
index = LoadedIndex(spec, KernelAdapter(spec))
|
| 373 |
+
index.load()
|
| 374 |
+
assert index.ready.is_set(), index.error
|
| 375 |
+
return index
|
| 376 |
+
|
| 377 |
+
|
| 378 |
+
def load_containers(tmp_path):
|
| 379 |
+
from bsides.search_indexes import ContainerAdapter
|
| 380 |
+
|
| 381 |
+
write_container_db(tmp_path, CONTAINER_ROWS)
|
| 382 |
+
write_matrix(tmp_path, basis(len(CONTAINER_ROWS)))
|
| 383 |
+
spec = make_spec(
|
| 384 |
+
tmp_path, key='docker', dataset_id='owner/containers',
|
| 385 |
+
table='containers', id_column='repo_id')
|
| 386 |
+
index = LoadedIndex(spec, ContainerAdapter(spec))
|
| 387 |
+
index.load()
|
| 388 |
+
assert index.ready.is_set(), index.error
|
| 389 |
+
return index
|
| 390 |
+
|
| 391 |
+
|
| 392 |
+
@pytest.fixture
|
| 393 |
+
def loaded_containers(tmp_path):
|
| 394 |
+
return load_containers(tmp_path)
|
| 395 |
+
|
| 396 |
+
|
| 397 |
+
class FakeEmbeddings:
|
| 398 |
+
def __init__(self, vector):
|
| 399 |
+
self.vector = [float(v) for v in vector]
|
| 400 |
+
self.calls: list[dict] = []
|
| 401 |
+
|
| 402 |
+
def create(self, **kwargs):
|
| 403 |
+
self.calls.append(kwargs)
|
| 404 |
+
return SimpleNamespace(
|
| 405 |
+
data=[SimpleNamespace(embedding=list(self.vector))])
|
| 406 |
+
|
| 407 |
+
|
| 408 |
+
class FakeClient:
|
| 409 |
+
def __init__(self, vector):
|
| 410 |
+
self.embeddings = FakeEmbeddings(vector)
|
| 411 |
+
|
| 412 |
+
|
| 413 |
+
# ββ download contract βββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 414 |
+
|
| 415 |
+
def test_download_index_requests_dataset_files_into_spec_dir(tmp_path):
|
| 416 |
+
spec = make_spec(tmp_path / 'idx')
|
| 417 |
+
calls: list[tuple[str, dict]] = []
|
| 418 |
+
|
| 419 |
+
def downloader(dataset_id, **kwargs):
|
| 420 |
+
calls.append((dataset_id, kwargs))
|
| 421 |
+
|
| 422 |
+
download_index(downloader, spec)
|
| 423 |
+
|
| 424 |
+
assert calls == [
|
| 425 |
+
('owner/corpus', {
|
| 426 |
+
'repo_type': 'dataset',
|
| 427 |
+
'local_dir': tmp_path / 'idx',
|
| 428 |
+
'allow_patterns': ['embeddings.f16.npy', 'meta.sqlite'],
|
| 429 |
+
})
|
| 430 |
+
]
|
| 431 |
+
|
| 432 |
+
|
| 433 |
+
# ββ loader: happy paths βββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 434 |
+
|
| 435 |
+
def test_load_validates_marks_ready_and_computes_facets(loaded_models):
|
| 436 |
+
assert loaded_models.error is None
|
| 437 |
+
assert loaded_models.rows == 3
|
| 438 |
+
assert loaded_models.matrix.shape == (3, 4)
|
| 439 |
+
assert loaded_models.matrix.dtype == np.float32
|
| 440 |
+
assert loaded_models.facets['total'] == 3
|
| 441 |
+
|
| 442 |
+
|
| 443 |
+
def test_load_pulls_dataset_when_files_are_missing(tmp_path):
|
| 444 |
+
src = tmp_path / 'src'
|
| 445 |
+
src.mkdir()
|
| 446 |
+
write_index_dir(src, [model_row(0, 'a/b')], basis(1))
|
| 447 |
+
spec = make_spec(tmp_path / 'idx')
|
| 448 |
+
seen: list[tuple[str, dict]] = []
|
| 449 |
+
|
| 450 |
+
def downloader(dataset_id, **kwargs):
|
| 451 |
+
seen.append((dataset_id, dict(kwargs)))
|
| 452 |
+
target = Path(kwargs['local_dir'])
|
| 453 |
+
target.mkdir(parents=True, exist_ok=True)
|
| 454 |
+
for name in kwargs['allow_patterns']:
|
| 455 |
+
(target / name).write_bytes((src / name).read_bytes())
|
| 456 |
+
|
| 457 |
+
index = LoadedIndex(spec, ModelAdapter(spec))
|
| 458 |
+
index.load(downloader)
|
| 459 |
+
|
| 460 |
+
assert seen and seen[0][0] == 'owner/corpus'
|
| 461 |
+
assert index.ready.is_set() and index.error is None and index.rows == 1
|
| 462 |
+
|
| 463 |
+
|
| 464 |
+
def test_load_without_downloader_never_imports_hub_when_files_exist(tmp_path):
|
| 465 |
+
# Files already present β the lazy huggingface_hub import never runs.
|
| 466 |
+
write_index_dir(tmp_path, [model_row(0, 'a/b')], basis(1))
|
| 467 |
+
index = LoadedIndex(make_spec(tmp_path))
|
| 468 |
+
index.load()
|
| 469 |
+
assert index.ready.is_set()
|
| 470 |
+
|
| 471 |
+
|
| 472 |
+
|
| 473 |
+
# ββ loader: validation failures stay isolated βββββββββββββββββββββββββββββββββ
|
| 474 |
+
|
| 475 |
+
def test_load_rejects_matrix_rank_not_two(tmp_path):
|
| 476 |
+
write_model_db(tmp_path, [model_row(0, 'a/b')])
|
| 477 |
+
np.save(tmp_path / MATRIX_NAME, np.zeros(4, dtype=np.float16)) # 1-D
|
| 478 |
+
index = LoadedIndex(make_spec(tmp_path))
|
| 479 |
+
index.load()
|
| 480 |
+
assert not index.ready.is_set()
|
| 481 |
+
assert index.matrix is None
|
| 482 |
+
assert index.error and 'rank' in index.error
|
| 483 |
+
|
| 484 |
+
|
| 485 |
+
def test_load_rejects_dimension_mismatch(tmp_path):
|
| 486 |
+
write_model_db(tmp_path, [model_row(0, 'a/b')])
|
| 487 |
+
write_matrix(tmp_path, basis(1, dim=8)) # spec expects 4
|
| 488 |
+
index = LoadedIndex(make_spec(tmp_path))
|
| 489 |
+
index.load()
|
| 490 |
+
assert not index.ready.is_set()
|
| 491 |
+
assert index.error and 'dimensions' in index.error
|
| 492 |
+
|
| 493 |
+
|
| 494 |
+
def test_load_rejects_non_finite_values(tmp_path):
|
| 495 |
+
write_model_db(tmp_path, [model_row(0, 'a/b')])
|
| 496 |
+
m = basis(1)
|
| 497 |
+
m[0, 0] = np.inf
|
| 498 |
+
write_matrix(tmp_path, m)
|
| 499 |
+
index = LoadedIndex(make_spec(tmp_path))
|
| 500 |
+
index.load()
|
| 501 |
+
assert not index.ready.is_set()
|
| 502 |
+
assert index.error and 'non-finite' in index.error
|
| 503 |
+
|
| 504 |
+
|
| 505 |
+
def test_load_rejects_sqlite_that_fails_quick_check(tmp_path):
|
| 506 |
+
(tmp_path / SQLITE_NAME).write_bytes(b'this is not a sqlite database')
|
| 507 |
+
write_matrix(tmp_path, basis(1))
|
| 508 |
+
index = LoadedIndex(make_spec(tmp_path))
|
| 509 |
+
index.load()
|
| 510 |
+
assert not index.ready.is_set()
|
| 511 |
+
assert index.error
|
| 512 |
+
|
| 513 |
+
|
| 514 |
+
def test_load_rejects_row_count_mismatch(tmp_path):
|
| 515 |
+
write_model_db(tmp_path, [model_row(0, 'a/b')])
|
| 516 |
+
write_matrix(tmp_path, basis(2))
|
| 517 |
+
index = LoadedIndex(make_spec(tmp_path))
|
| 518 |
+
index.load()
|
| 519 |
+
assert not index.ready.is_set()
|
| 520 |
+
assert index.error and 'rows' in index.error
|
| 521 |
+
|
| 522 |
+
|
| 523 |
+
def test_load_rejects_rows_outside_matrix_bounds(tmp_path):
|
| 524 |
+
write_model_db(tmp_path,
|
| 525 |
+
[model_row(0, 'a/b'), model_row(5, 'c/d')]) # row 5 >= N=2
|
| 526 |
+
write_matrix(tmp_path, basis(2))
|
| 527 |
+
index = LoadedIndex(make_spec(tmp_path))
|
| 528 |
+
index.load()
|
| 529 |
+
assert not index.ready.is_set()
|
| 530 |
+
assert index.error and 'bounds' in index.error
|
| 531 |
+
|
| 532 |
+
|
| 533 |
+
def test_load_accepts_empty_index(tmp_path):
|
| 534 |
+
write_model_db(tmp_path, [])
|
| 535 |
+
write_matrix(tmp_path, np.zeros((0, 4), dtype=np.float32))
|
| 536 |
+
index = LoadedIndex(make_spec(tmp_path))
|
| 537 |
+
index.load()
|
| 538 |
+
assert index.ready.is_set() and index.rows == 0
|
| 539 |
+
|
| 540 |
+
|
| 541 |
+
def test_load_isolates_downloader_failure(tmp_path):
|
| 542 |
+
def downloader(dataset_id, **kwargs):
|
| 543 |
+
raise RuntimeError('hub says no')
|
| 544 |
+
|
| 545 |
+
index = LoadedIndex(make_spec(tmp_path))
|
| 546 |
+
index.load(downloader)
|
| 547 |
+
assert not index.ready.is_set()
|
| 548 |
+
assert index.error and 'hub says no' in index.error
|
| 549 |
+
|
| 550 |
+
|
| 551 |
+
|
| 552 |
+
# ββ reload: hot-swap contract βββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 553 |
+
|
| 554 |
+
def republish(dir_path, rows, vectors):
|
| 555 |
+
"""A downloader that replaces the corpus files, like a fresh publish."""
|
| 556 |
+
def downloader(dataset_id, **kwargs):
|
| 557 |
+
target = Path(kwargs['local_dir'])
|
| 558 |
+
for name in (SQLITE_NAME, MATRIX_NAME):
|
| 559 |
+
(target / name).unlink(missing_ok=True)
|
| 560 |
+
write_index_dir(target, rows, vectors)
|
| 561 |
+
return downloader
|
| 562 |
+
|
| 563 |
+
|
| 564 |
+
def test_reload_swaps_in_the_freshly_published_index(loaded_models):
|
| 565 |
+
assert loaded_models.rows == 3
|
| 566 |
+
|
| 567 |
+
swapped = loaded_models.reload(republish(
|
| 568 |
+
loaded_models.spec.local_dir,
|
| 569 |
+
[model_row(0, 'indie/delta', tags='["rwkv"]', model_type='rwkv'),
|
| 570 |
+
model_row(1, 'indie/epsilon', tags='["mamba"]', model_type='mamba')],
|
| 571 |
+
basis(2)))
|
| 572 |
+
|
| 573 |
+
assert swapped is True
|
| 574 |
+
assert loaded_models.error is None
|
| 575 |
+
assert loaded_models.ready.is_set()
|
| 576 |
+
assert loaded_models.rows == 2
|
| 577 |
+
assert loaded_models.matrix.shape == (2, 4)
|
| 578 |
+
ids = [r[0] for r in loaded_models.db().execute(
|
| 579 |
+
'SELECT model_id FROM models ORDER BY row')]
|
| 580 |
+
assert ids == ['indie/delta', 'indie/epsilon']
|
| 581 |
+
assert loaded_models.facets['total'] == 2
|
| 582 |
+
assert {a['v'] for a in loaded_models.facets['archs']} == {'rwkv', 'mamba'}
|
| 583 |
+
|
| 584 |
+
|
| 585 |
+
def test_reload_invalidates_connections_to_the_replaced_file(loaded_models):
|
| 586 |
+
stale = loaded_models.db()
|
| 587 |
+
generation = loaded_models.generation
|
| 588 |
+
|
| 589 |
+
loaded_models.reload(republish(
|
| 590 |
+
loaded_models.spec.local_dir,
|
| 591 |
+
[model_row(0, 'indie/delta')], basis(1)))
|
| 592 |
+
|
| 593 |
+
assert loaded_models.generation > generation
|
| 594 |
+
fresh = loaded_models.db()
|
| 595 |
+
assert fresh is not stale
|
| 596 |
+
assert fresh.execute(
|
| 597 |
+
'SELECT model_id FROM models').fetchone()[0] == 'indie/delta'
|
| 598 |
+
|
| 599 |
+
|
| 600 |
+
def test_reload_does_not_skip_the_download_step(loaded_models):
|
| 601 |
+
# load() skips the downloader call when files exist; reload() must not, or
|
| 602 |
+
# a newly published revision would never be checked for changed files.
|
| 603 |
+
seen: list[str] = []
|
| 604 |
+
|
| 605 |
+
def downloader(dataset_id, **kwargs):
|
| 606 |
+
seen.append(dataset_id)
|
| 607 |
+
|
| 608 |
+
assert loaded_models.reload(downloader) is True
|
| 609 |
+
assert seen == ['owner/corpus']
|
| 610 |
+
|
| 611 |
+
|
| 612 |
+
def test_failed_download_keeps_serving_the_previous_index(loaded_models):
|
| 613 |
+
def downloader(dataset_id, **kwargs):
|
| 614 |
+
raise RuntimeError('hub says no')
|
| 615 |
+
|
| 616 |
+
assert loaded_models.reload(downloader) is False
|
| 617 |
+
assert loaded_models.ready.is_set() # never went down
|
| 618 |
+
assert loaded_models.rows == 3
|
| 619 |
+
assert loaded_models.matrix.shape == (3, 4)
|
| 620 |
+
assert loaded_models.facets['total'] == 3
|
| 621 |
+
assert 'hub says no' in loaded_models.error
|
| 622 |
+
assert loaded_models.db().execute(
|
| 623 |
+
'SELECT COUNT(*) FROM models').fetchone()[0] == 3
|
| 624 |
+
|
| 625 |
+
|
| 626 |
+
def test_reload_rejecting_a_bad_publish_keeps_serving_the_old_index(
|
| 627 |
+
loaded_models):
|
| 628 |
+
# A published matrix at the wrong width must not replace a good one.
|
| 629 |
+
assert loaded_models.reload(republish(
|
| 630 |
+
loaded_models.spec.local_dir,
|
| 631 |
+
[model_row(0, 'indie/delta')], basis(1, dim=8))) is False
|
| 632 |
+
|
| 633 |
+
assert loaded_models.ready.is_set()
|
| 634 |
+
assert loaded_models.rows == 3
|
| 635 |
+
assert loaded_models.matrix.shape == (3, 4)
|
| 636 |
+
assert 'dimensions' in loaded_models.error
|
| 637 |
+
|
| 638 |
+
|
| 639 |
+
def test_reload_rejects_a_publish_whose_rows_do_not_match_the_matrix(
|
| 640 |
+
loaded_models):
|
| 641 |
+
assert loaded_models.reload(republish(
|
| 642 |
+
loaded_models.spec.local_dir,
|
| 643 |
+
[model_row(0, 'indie/delta')], basis(2))) is False
|
| 644 |
+
|
| 645 |
+
assert loaded_models.ready.is_set() and loaded_models.rows == 3
|
| 646 |
+
assert 'rows' in loaded_models.error
|
| 647 |
+
|
| 648 |
+
|
| 649 |
+
# ββ registry ββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 650 |
+
|
| 651 |
+
def test_registry_isolates_a_broken_corpus_from_a_healthy_one(tmp_path):
|
| 652 |
+
good_dir, bad_dir = tmp_path / 'good', tmp_path / 'bad'
|
| 653 |
+
good_dir.mkdir()
|
| 654 |
+
bad_dir.mkdir()
|
| 655 |
+
write_index_dir(good_dir, FIXTURE_ROWS, basis(3))
|
| 656 |
+
write_model_db(bad_dir, [model_row(0, 'a/b')])
|
| 657 |
+
write_matrix(bad_dir, basis(1, dim=8)) # wrong dims β validation failure
|
| 658 |
+
|
| 659 |
+
registry = IndexRegistry()
|
| 660 |
+
registry.register(ModelAdapter(make_spec(good_dir, key='models')))
|
| 661 |
+
registry.register(ModelAdapter(make_spec(bad_dir, key='kernels')))
|
| 662 |
+
registry.load_all() # must not raise
|
| 663 |
+
|
| 664 |
+
assert registry.index('models').ready.is_set()
|
| 665 |
+
bad = registry.index('kernels')
|
| 666 |
+
assert not bad.ready.is_set() and bad.error
|
| 667 |
+
|
| 668 |
+
status = registry.status()
|
| 669 |
+
assert status['models']['ready'] is True
|
| 670 |
+
assert status['models']['rows'] == 3
|
| 671 |
+
assert status['models']['error'] is None
|
| 672 |
+
assert status['kernels']['ready'] is False
|
| 673 |
+
assert status['kernels']['error']
|
| 674 |
+
|
| 675 |
+
|
| 676 |
+
def test_registry_unknown_key_raises_keyerror():
|
| 677 |
+
with pytest.raises(KeyError):
|
| 678 |
+
IndexRegistry().index('bogus')
|
| 679 |
+
|
| 680 |
+
|
| 681 |
+
def test_registry_start_loading_runs_isolated_threads(tmp_path):
|
| 682 |
+
good_dir = tmp_path / 'good'
|
| 683 |
+
good_dir.mkdir()
|
| 684 |
+
write_index_dir(good_dir, FIXTURE_ROWS, basis(3))
|
| 685 |
+
registry = IndexRegistry()
|
| 686 |
+
registry.register(ModelAdapter(make_spec(good_dir, key='models')))
|
| 687 |
+
registry.start_loading()
|
| 688 |
+
index = registry.index('models')
|
| 689 |
+
assert index.ready.wait(timeout=10)
|
| 690 |
+
assert index.error is None
|
| 691 |
+
|
| 692 |
+
|
| 693 |
+
# ββ LoadedIndex connections βββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 694 |
+
|
| 695 |
+
def test_db_is_thread_local_readonly_and_generation_scoped(loaded_models):
|
| 696 |
+
conn = loaded_models.db()
|
| 697 |
+
assert conn is loaded_models.db() # same thread β same connection
|
| 698 |
+
with pytest.raises(sqlite3.OperationalError):
|
| 699 |
+
conn.execute('CREATE TABLE hack (x INTEGER)')
|
| 700 |
+
|
| 701 |
+
loaded_models.invalidate_connections()
|
| 702 |
+
fresh = loaded_models.db()
|
| 703 |
+
assert fresh is not conn
|
| 704 |
+
assert fresh.execute('SELECT COUNT(*) FROM models').fetchone()[0] == 3
|
| 705 |
+
|
| 706 |
+
|
| 707 |
+
# ββ query embedding βββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 708 |
+
|
| 709 |
+
def test_embed_query_uses_spec_model_and_selected_matrix_dimensions(
|
| 710 |
+
loaded_models):
|
| 711 |
+
client = FakeClient([2.0, 0.0, 0.0, 0.0])
|
| 712 |
+
vector = embed_query(loaded_models, client, 'odd attention')
|
| 713 |
+
|
| 714 |
+
assert vector.shape == (4,)
|
| 715 |
+
assert np.isclose(np.linalg.norm(vector), 1.0)
|
| 716 |
+
assert client.embeddings.calls == [{
|
| 717 |
+
'model': 'text-embedding-3-small',
|
| 718 |
+
'dimensions': 4,
|
| 719 |
+
'input': 'odd attention',
|
| 720 |
+
}]
|
| 721 |
+
|
| 722 |
+
|
| 723 |
+
def test_embed_query_caps_input_at_max_chars(loaded_models):
|
| 724 |
+
client = FakeClient([1, 0, 0, 0])
|
| 725 |
+
embed_query(loaded_models, client, 'x' * (MAX_QUERY_CHARS + 250))
|
| 726 |
+
assert len(client.embeddings.calls[0]['input']) == MAX_QUERY_CHARS
|
| 727 |
+
|
| 728 |
+
|
| 729 |
+
def test_embed_query_without_client_names_the_selected_credential(loaded_models):
|
| 730 |
+
with pytest.raises(
|
| 731 |
+
UnavailableError, match='no OPENAI_KERNEL_API_KEY'):
|
| 732 |
+
embed_query(
|
| 733 |
+
loaded_models, None, 'q',
|
| 734 |
+
credential_name='OPENAI_KERNEL_API_KEY')
|
| 735 |
+
|
| 736 |
+
|
| 737 |
+
def test_embed_query_without_matrix_is_unavailable(tmp_path):
|
| 738 |
+
index = LoadedIndex(make_spec(tmp_path))
|
| 739 |
+
with pytest.raises(UnavailableError, match='index not loaded'):
|
| 740 |
+
embed_query(index, FakeClient([1, 0, 0, 0]), 'q')
|
| 741 |
+
|
| 742 |
+
|
| 743 |
+
|
| 744 |
+
# ββ shared semantic ranking βββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 745 |
+
|
| 746 |
+
def test_semantic_rank_scores_orders_and_paginates(loaded_models):
|
| 747 |
+
client = FakeClient([0.2, 1.0, 0.5, 0.0]) # beta > gamma > alpha
|
| 748 |
+
total, cards = semantic_rank(
|
| 749 |
+
loaded_models.db(), loaded_models, loaded_models.adapter,
|
| 750 |
+
where='1=1', args=[], sem_q='mamba hybrid', page=0, page_size=2,
|
| 751 |
+
embed_client=client)
|
| 752 |
+
|
| 753 |
+
assert total == 3
|
| 754 |
+
assert [c['model_id'] for c in cards] == ['indie/beta', 'indie/gamma']
|
| 755 |
+
assert cards[0]['score'] > cards[1]['score']
|
| 756 |
+
assert cards[0]['tags'] == ['mamba'] # JSON columns decoded
|
| 757 |
+
|
| 758 |
+
|
| 759 |
+
def test_semantic_rank_uses_candidate_scoring_for_small_filtered_sets(
|
| 760 |
+
loaded_models):
|
| 761 |
+
# total=1 < N//2 β MAT[rows] @ qv branch instead of (MAT @ qv)[rows]
|
| 762 |
+
client = FakeClient([0.0, 0.0, 1.0, 0.0])
|
| 763 |
+
total, cards = semantic_rank(
|
| 764 |
+
loaded_models.db(), loaded_models, loaded_models.adapter,
|
| 765 |
+
where='model_id = ?', args=['indie/gamma'], sem_q='q',
|
| 766 |
+
page=0, page_size=24, embed_client=client)
|
| 767 |
+
|
| 768 |
+
assert total == 1
|
| 769 |
+
assert [c['model_id'] for c in cards] == ['indie/gamma']
|
| 770 |
+
assert cards[0]['score'] == 1.0
|
| 771 |
+
|
| 772 |
+
|
| 773 |
+
def test_semantic_rank_empty_candidates_skip_embedding(loaded_models):
|
| 774 |
+
client = FakeClient([1, 0, 0, 0])
|
| 775 |
+
total, cards = semantic_rank(
|
| 776 |
+
loaded_models.db(), loaded_models, loaded_models.adapter,
|
| 777 |
+
where='row < 0', args=[], sem_q='q', page=0, page_size=24,
|
| 778 |
+
embed_client=client)
|
| 779 |
+
|
| 780 |
+
assert (total, cards) == (0, [])
|
| 781 |
+
assert client.embeddings.calls == []
|
| 782 |
+
|
| 783 |
+
|
| 784 |
+
def test_semantic_rank_page_beyond_window_returns_total_but_no_cards(
|
| 785 |
+
loaded_models):
|
| 786 |
+
client = FakeClient([1, 0, 0, 0])
|
| 787 |
+
total, cards = semantic_rank(
|
| 788 |
+
loaded_models.db(), loaded_models, loaded_models.adapter,
|
| 789 |
+
where='1=1', args=[], sem_q='q', page=3, page_size=2,
|
| 790 |
+
embed_client=client)
|
| 791 |
+
|
| 792 |
+
assert total == 3 and cards == []
|
| 793 |
+
|
| 794 |
+
|
| 795 |
+
def test_semantic_rank_propagates_unavailable_embedding(loaded_models):
|
| 796 |
+
with pytest.raises(UnavailableError):
|
| 797 |
+
semantic_rank(
|
| 798 |
+
loaded_models.db(), loaded_models, loaded_models.adapter,
|
| 799 |
+
where='1=1', args=[], sem_q='q', page=0, page_size=2,
|
| 800 |
+
embed_client=None)
|
| 801 |
+
|
| 802 |
+
|
| 803 |
+
# ββ shared structured paging ββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 804 |
+
|
| 805 |
+
def test_structured_page_orders_counts_and_paginates(loaded_models):
|
| 806 |
+
total, cards = structured_page(
|
| 807 |
+
loaded_models.db(), loaded_models, loaded_models.adapter,
|
| 808 |
+
where='1=1', args=[], sort='downloads', page=0, page_size=2)
|
| 809 |
+
|
| 810 |
+
assert total == 3
|
| 811 |
+
assert [c['model_id'] for c in cards] == ['indie/beta', 'indie/gamma']
|
| 812 |
+
|
| 813 |
+
total2, cards2 = structured_page(
|
| 814 |
+
loaded_models.db(), loaded_models, loaded_models.adapter,
|
| 815 |
+
where='1=1', args=[], sort='downloads', page=1, page_size=2)
|
| 816 |
+
assert total2 == 3
|
| 817 |
+
assert [c['model_id'] for c in cards2] == ['indie/alpha']
|
| 818 |
+
|
| 819 |
+
|
| 820 |
+
def test_structured_page_unknown_sort_falls_back_to_newest(loaded_models):
|
| 821 |
+
total, cards = structured_page(
|
| 822 |
+
loaded_models.db(), loaded_models, loaded_models.adapter,
|
| 823 |
+
where='1=1', args=[], sort='relevance', page=0, page_size=24)
|
| 824 |
+
|
| 825 |
+
assert total == 3
|
| 826 |
+
assert [c['model_id'] for c in cards] == [
|
| 827 |
+
'indie/beta', 'indie/gamma', 'indie/alpha'] # created_at DESC
|
| 828 |
+
|
| 829 |
+
|
| 830 |
+
|
| 831 |
+
# ββ ModelAdapter: request parsing βββββββββββββββββββββββββββββββββββββββββββββ
|
| 832 |
+
|
| 833 |
+
def test_extract_constraints_pulls_dates_params_downloads_and_flags(
|
| 834 |
+
loaded_models):
|
| 835 |
+
r = SearchReq(query='mamba hybrid feb 2025 under 3b params '
|
| 836 |
+
'10-50 downloads dpo custom code')
|
| 837 |
+
sem_q = loaded_models.adapter.extract_constraints(r)
|
| 838 |
+
|
| 839 |
+
assert r.month_from == r.month_to == '2025-02'
|
| 840 |
+
assert r.params_max == 3_000_000_000
|
| 841 |
+
assert r.downloads_min == 10 and r.downloads_max == 50
|
| 842 |
+
assert r.custom_code_only is True
|
| 843 |
+
assert 'dpo' in r.methods
|
| 844 |
+
# constraints leave the embedding text; meaning stays
|
| 845 |
+
assert 'feb' not in sem_q and '2025' not in sem_q
|
| 846 |
+
assert 'mamba' in sem_q and 'hybrid' in sem_q and 'dpo' in sem_q
|
| 847 |
+
|
| 848 |
+
|
| 849 |
+
def test_extract_constraints_never_loosens_explicit_filters(loaded_models):
|
| 850 |
+
r = SearchReq(query='2024 mamba', month_from='2023-06', methods=['sft'])
|
| 851 |
+
loaded_models.adapter.extract_constraints(r)
|
| 852 |
+
|
| 853 |
+
assert r.month_from == '2023-06' # explicit value untouched
|
| 854 |
+
assert r.month_to == '2024-12' # empty slot filled
|
| 855 |
+
assert r.methods == ['sft']
|
| 856 |
+
|
| 857 |
+
|
| 858 |
+
def test_extract_constraints_empty_remainder_skips_embedding(loaded_models):
|
| 859 |
+
r = SearchReq(query='under 3b params')
|
| 860 |
+
assert loaded_models.adapter.extract_constraints(r) == ''
|
| 861 |
+
assert r.params_max == 3_000_000_000
|
| 862 |
+
|
| 863 |
+
|
| 864 |
+
# ββ ModelAdapter: filtering, serialization, facets ββββββββββββββββββββββββββββ
|
| 865 |
+
|
| 866 |
+
def test_model_adapter_exposes_spec_result_key_and_card_columns(loaded_models):
|
| 867 |
+
adapter = loaded_models.adapter
|
| 868 |
+
assert adapter.result_key == 'models'
|
| 869 |
+
assert adapter.card_columns == (
|
| 870 |
+
'row, model_id, author, month, downloads, likes, tags, base_model, '
|
| 871 |
+
'relation, license, params, model_type, architectures, context_len, '
|
| 872 |
+
'num_experts, code_imports, gated')
|
| 873 |
+
assert adapter.sort_order('downloads') == 'downloads DESC'
|
| 874 |
+
assert adapter.sort_order('bogus') == 'created_at DESC'
|
| 875 |
+
|
| 876 |
+
|
| 877 |
+
def test_build_where_empty_request_is_neutral(loaded_models):
|
| 878 |
+
where, args = loaded_models.adapter.build_where(SearchReq())
|
| 879 |
+
assert where == '1=1' and args == []
|
| 880 |
+
|
| 881 |
+
|
| 882 |
+
def test_model_exclusions_match_family_evidence(loaded_exclusion_models):
|
| 883 |
+
request = SearchReq(exclude_families=['llama', 'qwen'])
|
| 884 |
+
where, args = loaded_exclusion_models.adapter.build_where(request)
|
| 885 |
+
ids = [r[0] for r in loaded_exclusion_models.db().execute(
|
| 886 |
+
f'SELECT model_id FROM models WHERE {where}', args)]
|
| 887 |
+
|
| 888 |
+
assert 'indie/llama-derivative' not in ids
|
| 889 |
+
assert 'indie/qwen-tagged' not in ids
|
| 890 |
+
assert 'indie/mistral-architecture' in ids
|
| 891 |
+
|
| 892 |
+
|
| 893 |
+
def test_model_family_exclusion_matches_expanded_mistral_terms(
|
| 894 |
+
loaded_exclusion_models):
|
| 895 |
+
where, args = loaded_exclusion_models.adapter.build_where(
|
| 896 |
+
SearchReq(exclude_families=['mistral']))
|
| 897 |
+
ids = [r[0] for r in loaded_exclusion_models.db().execute(
|
| 898 |
+
f'SELECT model_id FROM models WHERE {where}', args)]
|
| 899 |
+
|
| 900 |
+
assert 'indie/mistral-architecture' not in ids
|
| 901 |
+
|
| 902 |
+
|
| 903 |
+
def test_quantization_exclusion_requires_strong_evidence(
|
| 904 |
+
loaded_exclusion_models):
|
| 905 |
+
request = SearchReq(exclude_quantizations=True)
|
| 906 |
+
where, args = loaded_exclusion_models.adapter.build_where(request)
|
| 907 |
+
ids = [r[0] for r in loaded_exclusion_models.db().execute(
|
| 908 |
+
f'SELECT model_id FROM models WHERE {where}', args)]
|
| 909 |
+
|
| 910 |
+
assert 'indie/gptq-repack' not in ids
|
| 911 |
+
assert 'indie/native-fp8-research' in ids
|
| 912 |
+
assert 'indie/native-int8-research' in ids
|
| 913 |
+
assert 'indie/config-null-quant' in ids
|
| 914 |
+
assert 'indie/malformed-config' in ids
|
| 915 |
+
|
| 916 |
+
|
| 917 |
+
def test_malformed_config_remains_searchable_during_quantization_exclusion(
|
| 918 |
+
loaded_exclusion_models):
|
| 919 |
+
request = SearchReq(exclude_quantizations=True)
|
| 920 |
+
where, args = loaded_exclusion_models.adapter.build_where(request)
|
| 921 |
+
assert (
|
| 922 |
+
"CASE WHEN json_valid(config) "
|
| 923 |
+
"THEN json_extract(config, '$.quantization_config') IS NOT NULL "
|
| 924 |
+
"ELSE 0 END" in where)
|
| 925 |
+
ids = [r[0] for r in loaded_exclusion_models.db().execute(
|
| 926 |
+
f'SELECT model_id FROM models WHERE {where}', args)]
|
| 927 |
+
|
| 928 |
+
assert 'indie/malformed-config' in ids
|
| 929 |
+
|
| 930 |
+
|
| 931 |
+
def test_model_exclusions_are_neutral_by_default(loaded_models):
|
| 932 |
+
request = SearchReq()
|
| 933 |
+
|
| 934 |
+
assert request.exclude_families == []
|
| 935 |
+
assert request.exclude_quantizations is False
|
| 936 |
+
where, args = loaded_models.adapter.build_where(request)
|
| 937 |
+
assert where == '1=1'
|
| 938 |
+
assert args == []
|
| 939 |
+
|
| 940 |
+
|
| 941 |
+
def test_build_where_covers_every_model_filter(loaded_models):
|
| 942 |
+
r = SearchReq(
|
| 943 |
+
month_from='2024-01', month_to='2024-12', archs=['mamba', 'llama'],
|
| 944 |
+
methods=['sft'], relation='finetune', base_model='llama',
|
| 945 |
+
author='indie', license='mit', imports=['flash_attn'],
|
| 946 |
+
params_min=1, params_max=2, context_min=4096, downloads_min=5,
|
| 947 |
+
downloads_max=50, moe_only=True, custom_code_only=True,
|
| 948 |
+
config_contains='rope')
|
| 949 |
+
where, args = loaded_models.adapter.build_where(r)
|
| 950 |
+
|
| 951 |
+
assert where == (
|
| 952 |
+
'1=1 AND month >= ? AND month <= ? '
|
| 953 |
+
'AND ((model_type = ? OR tags LIKE ?) OR (model_type = ? OR tags LIKE ?)) '
|
| 954 |
+
'AND tags LIKE ? AND relation = ? AND base_model LIKE ? AND author LIKE ? '
|
| 955 |
+
'AND license = ? AND code_imports LIKE ? AND params >= ? AND params <= ? '
|
| 956 |
+
'AND context_len >= ? AND downloads >= ? AND downloads <= ? '
|
| 957 |
+
'AND num_experts > 1 '
|
| 958 |
+
"AND (code_imports IS NOT NULL OR tags LIKE '%\"custom_code\"%') "
|
| 959 |
+
'AND config LIKE ?')
|
| 960 |
+
assert args == ['2024-01', '2024-12', 'mamba', '%"mamba"%', 'llama',
|
| 961 |
+
'%"llama"%', '%"sft"%', 'finetune', '%llama%', '%indie%',
|
| 962 |
+
'mit', '%"flash_attn"%', 1, 2, 4096, 5, 50, '%rope%']
|
| 963 |
+
|
| 964 |
+
|
| 965 |
+
def test_build_where_executes_against_real_sqlite(loaded_models):
|
| 966 |
+
where, args = loaded_models.adapter.build_where(SearchReq(archs=['mamba']))
|
| 967 |
+
n = loaded_models.db().execute(
|
| 968 |
+
f'SELECT COUNT(*) FROM models WHERE {where}', args).fetchone()[0]
|
| 969 |
+
assert n == 1
|
| 970 |
+
|
| 971 |
+
|
| 972 |
+
def test_card_decodes_json_columns_and_rounds_score(loaded_models):
|
| 973 |
+
row = loaded_models.db().execute(
|
| 974 |
+
f'SELECT {loaded_models.adapter.card_columns} FROM models '
|
| 975 |
+
'WHERE row = 0').fetchone()
|
| 976 |
+
d = loaded_models.adapter.card(row, 0.87654)
|
| 977 |
+
|
| 978 |
+
assert d['tags'] == ['llama', 'sft']
|
| 979 |
+
assert d['code_imports'] == ['flash_attn']
|
| 980 |
+
assert d['architectures'] == ['LlamaForCausalLM']
|
| 981 |
+
assert d['score'] == 0.8765
|
| 982 |
+
|
| 983 |
+
|
| 984 |
+
def test_card_without_score_omits_it(loaded_models):
|
| 985 |
+
row = loaded_models.db().execute(
|
| 986 |
+
f'SELECT {loaded_models.adapter.card_columns} FROM models '
|
| 987 |
+
'WHERE row = 1').fetchone()
|
| 988 |
+
assert 'score' not in loaded_models.adapter.card(row)
|
| 989 |
+
|
| 990 |
+
|
| 991 |
+
def test_compute_facets_shape_matches_the_legacy_payload(loaded_models):
|
| 992 |
+
f = loaded_models.facets
|
| 993 |
+
|
| 994 |
+
assert f['total'] == 3
|
| 995 |
+
assert f['months'] == ['2024-01', '2024-03', '2025-02']
|
| 996 |
+
assert f['enriched'] is True
|
| 997 |
+
assert {a['v']: a['n'] for a in f['archs']} == {
|
| 998 |
+
'llama': 1, 'mamba': 1, 'qwen2': 1}
|
| 999 |
+
assert {m['v']: m['n'] for m in f['methods']} == {'sft': 1, 'dpo': 1}
|
| 1000 |
+
assert {l['v'] for l in f['licenses']} == {'mit', 'apache-2.0'}
|
| 1001 |
+
assert {r['v'] for r in f['relations']} == {'finetune'}
|
| 1002 |
+
assert {i['v'] for i in f['imports']} == {'flash_attn', 'triton'}
|
| 1003 |
+
assert {a['v'] for a in f['authors']} == {'indie', 'lab'}
|
| 1004 |
+
|
| 1005 |
+
|
| 1006 |
+
# ββ KernelAdapter: isolated request, filtering, cards and facets ββββββββββββββ
|
| 1007 |
+
|
| 1008 |
+
def test_search_request_keeps_optional_adapter_filters():
|
| 1009 |
+
request = SearchReq(filters={'languages': ['cuda']})
|
| 1010 |
+
|
| 1011 |
+
assert request.filters == {'languages': ['cuda']}
|
| 1012 |
+
|
| 1013 |
+
|
| 1014 |
+
def test_model_adapter_ignores_adapter_filters(loaded_models):
|
| 1015 |
+
request = SearchReq(filters={
|
| 1016 |
+
'languages': ['cuda'], 'has_build_dir': True,
|
| 1017 |
+
'variant_count_min': 99,
|
| 1018 |
+
})
|
| 1019 |
+
|
| 1020 |
+
where, args = loaded_models.adapter.build_where(request)
|
| 1021 |
+
|
| 1022 |
+
assert where == '1=1'
|
| 1023 |
+
assert args == []
|
| 1024 |
+
|
| 1025 |
+
|
| 1026 |
+
def test_kernel_filters_execute_against_real_sqlite(tmp_path):
|
| 1027 |
+
index = load_kernels(tmp_path)
|
| 1028 |
+
request = SearchReq(filters={
|
| 1029 |
+
'author': 'ali',
|
| 1030 |
+
'month_from': '2025-01',
|
| 1031 |
+
'month_to': '2025-01',
|
| 1032 |
+
'downloads_min': 50,
|
| 1033 |
+
'downloads_max': 200,
|
| 1034 |
+
'languages': ['cuda', 'triton'],
|
| 1035 |
+
'accelerators': ['nvidia'],
|
| 1036 |
+
'cuda_archs': ['sm_90'],
|
| 1037 |
+
'dtypes': ['bf16'],
|
| 1038 |
+
'intrinsics': ['mma_sync'],
|
| 1039 |
+
'torch_ops': ['matmul'],
|
| 1040 |
+
'torch_versions': ['2.5'],
|
| 1041 |
+
'has_build_dir': True,
|
| 1042 |
+
'variant_count_min': 3,
|
| 1043 |
+
'not_a_kernel_filter': 'ignored',
|
| 1044 |
+
})
|
| 1045 |
+
|
| 1046 |
+
where, args = index.adapter.build_where(request)
|
| 1047 |
+
ids = [row[0] for row in index.db().execute(
|
| 1048 |
+
f'SELECT repo_id FROM kernels WHERE {where}', args)]
|
| 1049 |
+
|
| 1050 |
+
assert ids == ['alice/fused-gemm']
|
| 1051 |
+
|
| 1052 |
+
|
| 1053 |
+
def test_kernel_array_filter_values_are_or_within_one_category(tmp_path):
|
| 1054 |
+
index = load_kernels(tmp_path)
|
| 1055 |
+
where, args = index.adapter.build_where(SearchReq(filters={
|
| 1056 |
+
'languages': ['cuda', 'triton'],
|
| 1057 |
+
}))
|
| 1058 |
+
|
| 1059 |
+
ids = [row[0] for row in index.db().execute(
|
| 1060 |
+
f'SELECT repo_id FROM kernels WHERE {where} ORDER BY row', args)]
|
| 1061 |
+
|
| 1062 |
+
assert ids == ['alice/fused-gemm', 'bob/paged-attention']
|
| 1063 |
+
|
| 1064 |
+
|
| 1065 |
+
def test_kernel_adapter_ignores_legacy_model_and_unknown_filters(tmp_path):
|
| 1066 |
+
index = load_kernels(tmp_path)
|
| 1067 |
+
request = SearchReq(
|
| 1068 |
+
author='nobody', downloads_min=9999, archs=['mamba'],
|
| 1069 |
+
filters={'license': 'mit', 'cpu_archs': ['x86_64']})
|
| 1070 |
+
|
| 1071 |
+
where, args = index.adapter.build_where(request)
|
| 1072 |
+
count = index.db().execute(
|
| 1073 |
+
f'SELECT COUNT(*) FROM kernels WHERE {where}', args).fetchone()[0]
|
| 1074 |
+
|
| 1075 |
+
assert count == 3
|
| 1076 |
+
|
| 1077 |
+
|
| 1078 |
+
def test_kernel_card_decodes_arrays_and_returns_facts(tmp_path):
|
| 1079 |
+
index = load_kernels(tmp_path)
|
| 1080 |
+
row = index.db().execute(
|
| 1081 |
+
f'SELECT {index.adapter.card_columns} FROM kernels WHERE row = 0'
|
| 1082 |
+
).fetchone()
|
| 1083 |
+
|
| 1084 |
+
card = index.adapter.card(row, 0.87654)
|
| 1085 |
+
|
| 1086 |
+
assert card['repo_id'] == 'alice/fused-gemm'
|
| 1087 |
+
assert card['tags'] == ['kernels']
|
| 1088 |
+
assert card['languages'] == ['cuda', 'python']
|
| 1089 |
+
assert card['accelerators'] == ['nvidia']
|
| 1090 |
+
assert card['torch_versions'] == ['2.5']
|
| 1091 |
+
assert card['cuda_archs'] == ['sm_90']
|
| 1092 |
+
assert card['cpu_archs'] == []
|
| 1093 |
+
assert card['operating_systems'] == ['linux']
|
| 1094 |
+
assert card['dtypes'] == ['bf16', 'fp16']
|
| 1095 |
+
assert card['intrinsics'] == ['mma_sync', '__syncthreads']
|
| 1096 |
+
assert card['kernel_names'] == ['fused_gemm']
|
| 1097 |
+
assert card['torch_ops'] == ['matmul']
|
| 1098 |
+
assert card['build_variants'] == ['torch25-cu124']
|
| 1099 |
+
assert card['variant_count'] == 3
|
| 1100 |
+
assert card['has_build_dir'] is True
|
| 1101 |
+
assert card['source_files'] == 4
|
| 1102 |
+
assert card['score'] == 0.8765
|
| 1103 |
+
assert 'document' not in card
|
| 1104 |
+
|
| 1105 |
+
|
| 1106 |
+
def test_kernel_facets_count_members_of_json_arrays(tmp_path):
|
| 1107 |
+
facets = load_kernels(tmp_path).facets
|
| 1108 |
+
|
| 1109 |
+
assert facets['total'] == 3
|
| 1110 |
+
assert facets['months'] == ['2024-06', '2025-01', '2025-02']
|
| 1111 |
+
assert {x['v']: x['n'] for x in facets['authors']} == {
|
| 1112 |
+
'alice': 2, 'bob': 1}
|
| 1113 |
+
assert {x['v']: x['n'] for x in facets['languages']} == {
|
| 1114 |
+
'python': 2, 'cpp': 1, 'cuda': 1, 'triton': 1}
|
| 1115 |
+
assert {x['v']: x['n'] for x in facets['accelerators']} == {
|
| 1116 |
+
'nvidia': 2, 'amd': 1}
|
| 1117 |
+
assert {x['v']: x['n'] for x in facets['torch_versions']} == {
|
| 1118 |
+
'2.5': 2, '2.4': 1}
|
| 1119 |
+
assert {x['v']: x['n'] for x in facets['cuda_archs']} == {
|
| 1120 |
+
'sm_80': 1, 'sm_90': 1}
|
| 1121 |
+
assert {x['v']: x['n'] for x in facets['dtypes']} == {
|
| 1122 |
+
'bf16': 1, 'fp16': 1, 'fp32': 1, 'int8': 1}
|
| 1123 |
+
assert {x['v']: x['n'] for x in facets['intrinsics']} == {
|
| 1124 |
+
'__syncthreads': 1, 'avx2': 1, 'mma_sync': 1, 'tl.dot': 1}
|
| 1125 |
+
assert {x['v']: x['n'] for x in facets['torch_ops']} == {
|
| 1126 |
+
'attention': 1, 'conv2d': 1, 'matmul': 1}
|
| 1127 |
+
assert facets['has_build_dir'] == [
|
| 1128 |
+
{'v': True, 'n': 2}, {'v': False, 'n': 1}]
|
| 1129 |
+
|
| 1130 |
+
|
| 1131 |
+
def test_kernel_structured_page_uses_kernel_envelope_contract(tmp_path):
|
| 1132 |
+
index = load_kernels(tmp_path)
|
| 1133 |
+
where, args = index.adapter.build_where(SearchReq(filters={
|
| 1134 |
+
'accelerators': ['nvidia'],
|
| 1135 |
+
}))
|
| 1136 |
+
|
| 1137 |
+
total, cards = structured_page(
|
| 1138 |
+
index.db(), index, index.adapter, where=where, args=args,
|
| 1139 |
+
sort='downloads', page=0, page_size=24)
|
| 1140 |
+
|
| 1141 |
+
assert index.adapter.result_key == 'kernels'
|
| 1142 |
+
assert total == 2
|
| 1143 |
+
assert [card['repo_id'] for card in cards] == [
|
| 1144 |
+
'alice/fused-gemm', 'bob/paged-attention']
|
| 1145 |
+
|
| 1146 |
+
|
| 1147 |
+
def test_kernel_query_text_is_preserved_for_semantic_search(tmp_path):
|
| 1148 |
+
index = load_kernels(tmp_path)
|
| 1149 |
+
|
| 1150 |
+
assert index.adapter.extract_constraints(
|
| 1151 |
+
SearchReq(query=' fused attention ')) == 'fused attention'
|
| 1152 |
+
|
| 1153 |
+
|
| 1154 |
+
# ββ ContainerAdapter: request, filtering, cards and facets βββββββββββββββββββ
|
| 1155 |
+
|
| 1156 |
+
def test_container_defaults_to_requested_recipe_filter(loaded_containers):
|
| 1157 |
+
request = SearchReq(filters={'artifact_kinds': ['recipe']})
|
| 1158 |
+
where, args = loaded_containers.adapter.build_where(request)
|
| 1159 |
+
kinds = {
|
| 1160 |
+
row[0] for row in loaded_containers.db().execute(
|
| 1161 |
+
f'SELECT artifact_kind FROM containers WHERE {where}', args)
|
| 1162 |
+
}
|
| 1163 |
+
|
| 1164 |
+
assert kinds == {'recipe'}
|
| 1165 |
+
|
| 1166 |
+
|
| 1167 |
+
def test_container_filters_are_anded_across_groups_and_ored_within_group(
|
| 1168 |
+
loaded_containers):
|
| 1169 |
+
request = SearchReq(filters={
|
| 1170 |
+
'artifact_kinds': ['recipe'],
|
| 1171 |
+
'python_versions': ['3.11', '3.12'],
|
| 1172 |
+
'accelerators': ['nvidia', 'amd'],
|
| 1173 |
+
'cuda_versions': ['12.8'],
|
| 1174 |
+
'frameworks': ['pytorch'],
|
| 1175 |
+
'has_multistage': True,
|
| 1176 |
+
})
|
| 1177 |
+
where, args = loaded_containers.adapter.build_where(request)
|
| 1178 |
+
ids = [row[0] for row in loaded_containers.db().execute(
|
| 1179 |
+
f'SELECT repo_id FROM containers WHERE {where}', args)]
|
| 1180 |
+
|
| 1181 |
+
assert ids == ['alice/cuda-recipe']
|
| 1182 |
+
|
| 1183 |
+
|
| 1184 |
+
def test_container_archive_rows_appear_when_artifact_kind_filter_is_absent(
|
| 1185 |
+
loaded_containers):
|
| 1186 |
+
request = SearchReq(filters={
|
| 1187 |
+
'python_versions': ['3.11'],
|
| 1188 |
+
'cuda_versions': ['12.8'],
|
| 1189 |
+
'frameworks': ['pytorch'],
|
| 1190 |
+
})
|
| 1191 |
+
where, args = loaded_containers.adapter.build_where(request)
|
| 1192 |
+
ids = [row[0] for row in loaded_containers.db().execute(
|
| 1193 |
+
f'SELECT repo_id FROM containers WHERE {where} ORDER BY row', args)]
|
| 1194 |
+
|
| 1195 |
+
assert ids == ['alice/cuda-recipe', 'dana/archive-bundle']
|
| 1196 |
+
|
| 1197 |
+
|
| 1198 |
+
def test_container_scalar_boolean_and_json_filters_execute_against_sqlite(
|
| 1199 |
+
loaded_containers):
|
| 1200 |
+
request = SearchReq(filters={
|
| 1201 |
+
'repo_types': ['dataset'],
|
| 1202 |
+
'services': ['jupyter', 'postgres'],
|
| 1203 |
+
'package_managers': ['apt'],
|
| 1204 |
+
'ports': ['8888'],
|
| 1205 |
+
'has_compose': True,
|
| 1206 |
+
'has_devcontainer': True,
|
| 1207 |
+
})
|
| 1208 |
+
where, args = loaded_containers.adapter.build_where(request)
|
| 1209 |
+
ids = [row[0] for row in loaded_containers.db().execute(
|
| 1210 |
+
f'SELECT repo_id FROM containers WHERE {where}', args)]
|
| 1211 |
+
|
| 1212 |
+
assert ids == ['carol/compose-stack']
|
| 1213 |
+
|
| 1214 |
+
|
| 1215 |
+
def test_container_query_text_is_preserved_for_semantic_search(
|
| 1216 |
+
loaded_containers):
|
| 1217 |
+
assert loaded_containers.adapter.extract_constraints(
|
| 1218 |
+
SearchReq(query=' python 3.11 cuda 12.8 ')
|
| 1219 |
+
) == 'python 3.11 cuda 12.8'
|
| 1220 |
+
|
| 1221 |
+
|
| 1222 |
+
def test_container_card_decodes_json_arrays_and_preserves_empty_arrays(
|
| 1223 |
+
loaded_containers):
|
| 1224 |
+
row = loaded_containers.db().execute(
|
| 1225 |
+
f'SELECT {loaded_containers.adapter.card_columns} FROM containers '
|
| 1226 |
+
'WHERE row = 1').fetchone()
|
| 1227 |
+
|
| 1228 |
+
card = loaded_containers.adapter.card(row, 0.87654)
|
| 1229 |
+
|
| 1230 |
+
assert card['repo_id'] == 'bob/rocm-recipe'
|
| 1231 |
+
assert card['repo_type'] == 'space'
|
| 1232 |
+
assert card['artifact_path'] == 'docker/Dockerfile'
|
| 1233 |
+
assert card['artifact_kind'] == 'recipe'
|
| 1234 |
+
assert card['tags'] == ['docker', 'rocm']
|
| 1235 |
+
assert card['artifact_files'] == ['docker/Dockerfile']
|
| 1236 |
+
assert card['qualification_reasons'] == ['dockerfile']
|
| 1237 |
+
assert card['base_images'] == ['rocm/pytorch:latest']
|
| 1238 |
+
assert card['accelerators'] == ['amd']
|
| 1239 |
+
assert card['cuda_versions'] == []
|
| 1240 |
+
assert card['rocm_versions'] == ['6.1']
|
| 1241 |
+
assert card['python_versions'] == ['3.12']
|
| 1242 |
+
assert card['node_versions'] == ['20']
|
| 1243 |
+
assert card['operating_systems'] == ['ubuntu24.04']
|
| 1244 |
+
assert card['ports'] == ['7860']
|
| 1245 |
+
assert card['services'] == []
|
| 1246 |
+
assert card['package_managers'] == ['apt', 'pip']
|
| 1247 |
+
assert card['frameworks'] == ['pytorch']
|
| 1248 |
+
assert card['entrypoints'] == []
|
| 1249 |
+
assert card['commands'] == ['python launch.py']
|
| 1250 |
+
assert card['has_compose'] is False
|
| 1251 |
+
assert card['has_devcontainer'] is False
|
| 1252 |
+
assert card['has_multistage'] is False
|
| 1253 |
+
assert card['score'] == 0.8765
|
| 1254 |
+
assert 'document' not in card
|
| 1255 |
+
|
| 1256 |
+
|
| 1257 |
+
def test_container_facets_count_scalars_arrays_and_booleans(
|
| 1258 |
+
loaded_containers):
|
| 1259 |
+
facets = loaded_containers.facets
|
| 1260 |
+
|
| 1261 |
+
assert facets['total'] == 4
|
| 1262 |
+
assert {item['v']: item['n'] for item in facets['repo_types']} == {
|
| 1263 |
+
'dataset': 2, 'model': 1, 'space': 1}
|
| 1264 |
+
assert {item['v']: item['n'] for item in facets['artifact_kinds']} == {
|
| 1265 |
+
'recipe': 3, 'archive': 1}
|
| 1266 |
+
assert {item['v']: item['n'] for item in facets['base_images']} == {
|
| 1267 |
+
'nvidia/cuda:12.8.0-runtime-ubuntu22.04': 1,
|
| 1268 |
+
'rocm/pytorch:latest': 1,
|
| 1269 |
+
'python:3.10-slim': 1,
|
| 1270 |
+
'ubuntu:22.04': 1,
|
| 1271 |
+
}
|
| 1272 |
+
assert {item['v']: item['n'] for item in facets['accelerators']} == {
|
| 1273 |
+
'nvidia': 2, 'amd': 1}
|
| 1274 |
+
assert {item['v']: item['n'] for item in facets['python_versions']} == {
|
| 1275 |
+
'3.11': 2, '3.12': 1, '3.10': 1}
|
| 1276 |
+
assert {item['v']: item['n'] for item in facets['frameworks']} == {
|
| 1277 |
+
'pytorch': 3, 'fastapi': 1}
|
| 1278 |
+
assert {item['v']: item['n'] for item in facets['services']} == {
|
| 1279 |
+
'api': 1, 'jupyter': 1, 'postgres': 1, 'worker': 1}
|
| 1280 |
+
assert {item['v']: item['n'] for item in facets['ports']} == {
|
| 1281 |
+
'7860': 2, '5432': 1, '8080': 1, '8888': 1}
|
| 1282 |
+
assert facets['has_compose'] == [
|
| 1283 |
+
{'v': True, 'n': 1}, {'v': False, 'n': 3}]
|
| 1284 |
+
assert facets['has_devcontainer'] == [
|
| 1285 |
+
{'v': True, 'n': 1}, {'v': False, 'n': 3}]
|
| 1286 |
+
assert facets['has_multistage'] == [
|
| 1287 |
+
{'v': True, 'n': 1}, {'v': False, 'n': 3}]
|