dnrops
/
meilisearch
mirror of https://github.com/meilisearch/meilisearch.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

wip

This commit is contained in:
ManyTheFish 2025-02-20 11:39:33 +01:00
parent 0464aa2fb0
commit 352d88b3df
21 changed files with 218 additions and 209 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

19
crates/filter-parser/src/condition.rs
View File

@ -30,6 +30,25 @@ pub enum Condition<'a> {
StartsWith { keyword: Token<'a>, word: Token<'a> },
}
impl Condition<'_> {
pub fn operator(&self) -> &str {
match self {
Condition::GreaterThan(_) => ">",
Condition::GreaterThanOrEqual(_) => ">=",
Condition::Equal(_) => "=",
Condition::NotEqual(_) => "!=",
Condition::Null => "IS NULL",
Condition::Empty => "IS EMPTY",
Condition::Exists => "EXISTS",
Condition::LowerThan(_) => "<",
Condition::LowerThanOrEqual(_) => "<=",
Condition::Between { .. } => "TO",
Condition::Contains { .. } => "CONTAINS",
Condition::StartsWith { .. } => "STARTS WITH",
}
}
}
/// condition = value ("==" | ">" ...) value
pub fn parse_condition(input: Span) -> IResult<FilterCondition> {
let operator = alt((tag("<="), tag(">="), tag("!="), tag("<"), tag(">"), tag("=")));

5
crates/index-scheduler/src/scheduler/test_failure.rs
View File

@ -1,11 +1,11 @@
use std::time::Instant;
use big_s::S;
use maplit::btreeset;
use meili_snap::snapshot;
use meilisearch_types::milli::obkv_to_json;
use meilisearch_types::milli::update::IndexDocumentsMethod::*;
use meilisearch_types::milli::update::Setting;
use meilisearch_types::milli::FilterableAttributesRule;
use meilisearch_types::tasks::KindWithContent;
use crate::insta_snapshot::snapshot_index_scheduler;
@ -127,7 +127,8 @@ fn fail_in_process_batch_for_document_deletion() {
use meilisearch_types::settings::{Settings, Unchecked};
let mut new_settings: Box<Settings<Unchecked>> = Box::default();
new_settings.filterable_attributes = Setting::Set(btreeset!(S("catto")));
new_settings.filterable_attributes =
Setting::Set(vec![FilterableAttributesRule::Field(S("catto"))]);
index_scheduler
.register(

1
crates/meilisearch-types/src/error.rs
View File

@ -399,6 +399,7 @@ impl ErrorCode for milli::Error {
UserError::AttributeLimitReached => Code::MaxFieldsLimitExceeded,
UserError::InvalidFilter(_) => Code::InvalidSearchFilter,
UserError::InvalidFilterExpression(..) => Code::InvalidSearchFilter,
UserError::FilterOperatorNotAllowed { .. } => Code::InvalidSearchFilter,
UserError::MissingDocumentId { .. } => Code::MissingDocumentId,
UserError::InvalidDocumentId { .. } | UserError::TooManyDocumentIds { .. } => {
Code::InvalidDocumentId

2
crates/meilisearch/src/routes/indexes/settings.rs
View File

@ -286,7 +286,7 @@ make_setting_routes!(
{
route: "/filterable-attributes",
update_verb: put,
value_type: std::collections::BTreeSet<String>,
value_type: Vec<meilisearch_types::milli::FilterableAttributesRule>,
err_type: meilisearch_types::deserr::DeserrJsonError<
meilisearch_types::error::deserr_codes::InvalidSettingsFilterableAttributes,
>,

7
crates/meilisearch/src/routes/indexes/settings_analytics.rs
View File

@ -8,6 +8,7 @@ use std::collections::{BTreeMap, BTreeSet, HashSet};
use meilisearch_types::facet_values_sort::FacetValuesSort;
use meilisearch_types::locales::{Locale, LocalizedAttributesRuleView};
use meilisearch_types::milli::update::Setting;
use meilisearch_types::milli::FilterableAttributesRule;
use meilisearch_types::settings::{
FacetingSettings, PaginationSettings, PrefixSearchSettings, ProximityPrecisionView,
RankingRuleView, SettingEmbeddingSettings, TypoSettings,
@ -331,10 +332,12 @@ pub struct FilterableAttributesAnalytics {
}
impl FilterableAttributesAnalytics {
pub fn new(setting: Option<&BTreeSet<String>>) -> Self {
pub fn new(setting: Option<&Vec<FilterableAttributesRule>>) -> Self {
Self {
total: setting.as_ref().map(|filter| filter.len()),
has_geo: setting.as_ref().map(|filter| filter.contains("_geo")),
has_geo: setting
.as_ref()
.map(|filter| filter.iter().any(FilterableAttributesRule::has_geo)),
}
}

9
crates/meilisearch/src/search/mod.rs
View File

@ -20,7 +20,7 @@ use meilisearch_types::milli::score_details::{ScoreDetails, ScoringStrategy};
use meilisearch_types::milli::vector::parsed_vectors::ExplicitVectors;
use meilisearch_types::milli::vector::Embedder;
use meilisearch_types::milli::{
FacetValueHit, InternalError, OrderBy, SearchForFacetValues, TimeBudget,
FacetValueHit, InternalError, OrderBy, PatternMatch, SearchForFacetValues, TimeBudget,
};
use meilisearch_types::settings::DEFAULT_PAGINATION_MAX_TOTAL_HITS;
use meilisearch_types::{milli, Document};
@ -1455,8 +1455,9 @@ pub fn perform_facet_search(
// If the facet string is not localized, we **ignore** the locales provided by the user because the facet data has no locale.
// If the user does not provide locales, we use the locales of the facet string.
let localized_attributes = index.localized_attributes_rules(&rtxn)?.unwrap_or_default();
let localized_attributes_locales =
localized_attributes.into_iter().find(|attr| attr.match_str(&facet_name));
let localized_attributes_locales = localized_attributes
.into_iter()
.find(|attr| attr.match_str(&facet_name) == PatternMatch::Match);
let locales = localized_attributes_locales.map(|attr| {
attr.locales
.into_iter()
@ -1802,7 +1803,7 @@ fn format_fields(
let locales = locales.or_else(|| {
localized_attributes
.iter()
.find(|rule| rule.match_str(key))
.find(|rule| rule.match_str(key) == PatternMatch::Match)
.map(LocalizedAttributesRule::locales)
});

6
crates/meilisearch/tests/common/server.rs
View File

@ -121,6 +121,12 @@ impl Server<Owned> {
self.service.post("/indexes", body).await
}
pub async fn delete_index(&self, uid: impl AsRef<str>) -> (Value, StatusCode) {
let url = format!("/indexes/{}", urlencoding::encode(uid.as_ref()));
let (value, code) = self.service.delete(url).await;
(value, code)
}
pub fn index_with_encoder(&self, uid: impl AsRef<str>, encoder: Encoder) -> Index<'_> {
Index {
uid: uid.as_ref().to_string(),

119
crates/meilisearch/tests/search/mod.rs
View File

@ -4,6 +4,7 @@
mod distinct;
mod errors;
mod facet_search;
mod filters;
mod formatted;
mod geo;
mod hybrid;
@ -16,12 +17,10 @@ mod restrict_searchable;
mod search_queue;
use meili_snap::{json_string, snapshot};
use meilisearch::Opt;
use tempfile::TempDir;
use crate::common::{
default_settings, shared_index_with_documents, shared_index_with_nested_documents, Server,
DOCUMENTS, FRUITS_DOCUMENTS, NESTED_DOCUMENTS, SCORE_DOCUMENTS, VECTOR_DOCUMENTS,
shared_index_with_documents, shared_index_with_nested_documents, Server, DOCUMENTS,
FRUITS_DOCUMENTS, NESTED_DOCUMENTS, SCORE_DOCUMENTS, VECTOR_DOCUMENTS,
};
use crate::json;
@ -321,118 +320,6 @@ async fn search_multiple_params() {
.await;
}
#[actix_rt::test]
async fn search_with_filter_string_notation() {
let server = Server::new().await;
let index = server.index("test");
let (_, code) = index.update_settings(json!({"filterableAttributes": ["title"]})).await;
meili_snap::snapshot!(code, @"202 Accepted");
let documents = DOCUMENTS.clone();
let (task, code) = index.add_documents(documents, None).await;
meili_snap::snapshot!(code, @"202 Accepted");
let res = index.wait_task(task.uid()).await;
meili_snap::snapshot!(res["status"], @r###""succeeded""###);
index
.search(
json!({
"filter": "title = Gläss"
}),
|response, code| {
assert_eq!(code, 200, "{}", response);
assert_eq!(response["hits"].as_array().unwrap().len(), 1);
},
)
.await;
let index = server.index("nested");
let (_, code) =
index.update_settings(json!({"filterableAttributes": ["cattos", "doggos.age"]})).await;
meili_snap::snapshot!(code, @"202 Accepted");
let documents = NESTED_DOCUMENTS.clone();
let (task, code) = index.add_documents(documents, None).await;
meili_snap::snapshot!(code, @"202 Accepted");
let res = index.wait_task(task.uid()).await;
meili_snap::snapshot!(res["status"], @r###""succeeded""###);
index
.search(
json!({
"filter": "cattos = pésti"
}),
|response, code| {
assert_eq!(code, 200, "{}", response);
assert_eq!(response["hits"].as_array().unwrap().len(), 1);
assert_eq!(response["hits"][0]["id"], json!(852));
},
)
.await;
index
.search(
json!({
"filter": "doggos.age > 5"
}),
|response, code| {
assert_eq!(code, 200, "{}", response);
assert_eq!(response["hits"].as_array().unwrap().len(), 2);
assert_eq!(response["hits"][0]["id"], json!(654));
assert_eq!(response["hits"][1]["id"], json!(951));
},
)
.await;
}
#[actix_rt::test]
async fn search_with_filter_array_notation() {
let index = shared_index_with_documents().await;
let (response, code) = index
.search_post(json!({
"filter": ["title = Gläss"]
}))
.await;
assert_eq!(code, 200, "{}", response);
assert_eq!(response["hits"].as_array().unwrap().len(), 1);
let (response, code) = index
.search_post(json!({
"filter": [["title = Gläss", "title = \"Shazam!\"", "title = \"Escape Room\""]]
}))
.await;
assert_eq!(code, 200, "{}", response);
assert_eq!(response["hits"].as_array().unwrap().len(), 3);
}
#[actix_rt::test]
async fn search_with_contains_filter() {
let temp = TempDir::new().unwrap();
let server = Server::new_with_options(Opt {
experimental_contains_filter: true,
..default_settings(temp.path())
})
.await
.unwrap();
let index = server.index("movies");
index.update_settings(json!({"filterableAttributes": ["title"]})).await;
let documents = DOCUMENTS.clone();
let (request, _code) = index.add_documents(documents, None).await;
index.wait_task(request.uid()).await.succeeded();
let (response, code) = index
.search_post(json!({
"filter": "title CONTAINS cap"
}))
.await;
assert_eq!(code, 200, "{}", response);
assert_eq!(response["hits"].as_array().unwrap().len(), 2);
}
#[actix_rt::test]
async fn search_with_sort_on_numbers() {
let index = shared_index_with_documents().await;

11
crates/milli/src/attribute_patterns.rs
View File

@ -4,7 +4,7 @@ use utoipa::ToSchema;
use crate::is_faceted_by;
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, Deserr, ToSchema)]
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, ToSchema)]
#[repr(transparent)]
#[serde(transparent)]
pub struct AttributePatterns {
@ -12,6 +12,15 @@ pub struct AttributePatterns {
pub patterns: Vec<String>,
}
impl<E: deserr::DeserializeError> Deserr<E> for AttributePatterns {
fn deserialize_from_value<V: deserr::IntoValue>(
value: deserr::Value<V>,
location: deserr::ValuePointerRef,
) -> Result<Self, E> {
Vec::<String>::deserialize_from_value(value, location).map(|patterns| Self { patterns })
}
}
impl From<Vec<String>> for AttributePatterns {
fn from(patterns: Vec<String>) -> Self {
Self { patterns }

2
crates/milli/src/error.rs
View File

@ -136,6 +136,8 @@ and can not be more than 511 bytes.", .document_id.to_string()
InvalidFilter(String),
#[error("Invalid type for filter subexpression: expected: {}, found: {}.", .0.join(", "), .1)]
InvalidFilterExpression(&'static [&'static str], Value),
#[error("Filter operator `{operator}` is not allowed for the attribute `{field}`, allowed operators: {}.", allowed_operators.join(", "))]
FilterOperatorNotAllowed { field: String, allowed_operators: Vec<String>, operator: String },
#[error("Attribute `{}` is not sortable. {}",
.field,
match .valid_fields.is_empty() {

6
crates/milli/src/fields_ids_map/metadata.rs
View File

@ -172,6 +172,12 @@ impl Metadata {
false
}
pub fn require_facet_level_database(&self, rules: &[FilterableAttributesRule]) -> bool {
let features = self.filterable_attributes_features(&rules);
self.is_sortable() || self.is_asc_desc() || features.is_filterable_comparison()
}
}
#[derive(Debug, Clone)]

119
crates/milli/src/filterable_attributes_rules.rs
View File

@ -41,7 +41,9 @@ impl FilterableAttributesRule {
#[deserr(rename_all = camelCase, deny_unknown_fields)]
pub struct FilterableAttributesPatterns {
pub patterns: AttributePatterns,
pub features: Option<FilterableAttributesFeatures>,
#[serde(default)]
#[deserr(default)]
pub features: FilterableAttributesFeatures,
}
impl FilterableAttributesPatterns {
@ -50,7 +52,7 @@ impl FilterableAttributesPatterns {
}
pub fn features(&self) -> FilterableAttributesFeatures {
self.features.clone().unwrap_or_default()
self.features.clone()
}
}
@ -59,52 +61,61 @@ impl FilterableAttributesPatterns {
#[deserr(rename_all = camelCase, deny_unknown_fields)]
pub struct FilterableAttributesFeatures {
facet_search: bool,
filter: FilterFeature,
filter: FilterFeatures,
}
impl Default for FilterableAttributesFeatures {
fn default() -> Self {
Self { facet_search: false, filter: FilterFeature::Equal }
Self { facet_search: false, filter: FilterFeatures::default() }
}
}
impl FilterableAttributesFeatures {
pub fn legacy_default() -> Self {
Self { facet_search: true, filter: FilterFeature::Order }
Self { facet_search: true, filter: FilterFeatures::legacy_default() }
}
pub fn no_features() -> Self {
Self { facet_search: false, filter: FilterFeature::Disabled }
Self { facet_search: false, filter: FilterFeatures::no_features() }
}
pub fn is_filterable(&self) -> bool {
self.filter != FilterFeature::Disabled
}
/// Check if `IS NULL` is allowed
pub fn is_filterable_null(&self) -> bool {
self.filter != FilterFeature::Disabled
self.filter.is_filterable()
}
/// Check if `IS EMPTY` is allowed
pub fn is_filterable_empty(&self) -> bool {
self.filter != FilterFeature::Disabled
self.filter.is_filterable_empty()
}
/// Check if `=` and `IN` are allowed
pub fn is_filterable_equality(&self) -> bool {
self.filter.is_filterable_equality()
}
/// Check if `IS NULL` is allowed
pub fn is_filterable_null(&self) -> bool {
self.filter.is_filterable_null()
}
/// Check if `IS EXISTS` is allowed
pub fn is_filterable_exists(&self) -> bool {
self.filter != FilterFeature::Disabled
self.filter.is_filterable_exists()
}
/// Check if `<`, `>`, `<=`, `>=` or `TO` are allowed
pub fn is_filterable_order(&self) -> bool {
self.filter == FilterFeature::Order
pub fn is_filterable_comparison(&self) -> bool {
self.filter.is_filterable_comparison()
}
/// Check if the facet search is allowed
pub fn is_facet_searchable(&self) -> bool {
self.facet_search
}
pub fn allowed_filter_operators(&self) -> Vec<String> {
self.filter.allowed_operators()
}
}
impl<E: DeserializeError> Deserr<E> for FilterableAttributesRule {
@ -123,10 +134,78 @@ impl<E: DeserializeError> Deserr<E> for FilterableAttributesRule {
}
#[derive(Serialize, Deserialize, PartialEq, Eq, Clone, Debug, Deserr, ToSchema)]
pub enum FilterFeature {
Disabled,
Equal,
Order,
pub struct FilterFeatures {
equality: bool,
comparison: bool,
}
impl FilterFeatures {
pub fn allowed_operators(&self) -> Vec<String> {
if !self.is_filterable() {
return vec![];
}
let mut operators = vec!["OR", "AND", "NOT"];
if self.is_filterable_equality() {
operators.extend_from_slice(&["=", "!=", "IN"]);
}
if self.is_filterable_comparison() {
operators.extend_from_slice(&["<", ">", "<=", ">=", "TO"]);
}
if self.is_filterable_empty() {
operators.push("IS EMPTY");
}
if self.is_filterable_null() {
operators.push("IS NULL");
}
if self.is_filterable_exists() {
operators.push("EXISTS");
}
operators.into_iter().map(String::from).collect()
}
pub fn is_filterable(&self) -> bool {
self.equality || self.comparison
}
pub fn is_filterable_equality(&self) -> bool {
self.equality
}
/// Check if `<`, `>`, `<=`, `>=` or `TO` are allowed
pub fn is_filterable_comparison(&self) -> bool {
self.comparison
}
/// Check if `IS EMPTY` is allowed
pub fn is_filterable_empty(&self) -> bool {
self.is_filterable()
}
/// Check if `IS EXISTS` is allowed
pub fn is_filterable_exists(&self) -> bool {
self.is_filterable()
}
/// Check if `IS NULL` is allowed
pub fn is_filterable_null(&self) -> bool {
self.is_filterable()
}
pub fn legacy_default() -> Self {
Self { equality: true, comparison: true }
}
pub fn no_features() -> Self {
Self { equality: false, comparison: false }
}
}
impl Default for FilterFeatures {
fn default() -> Self {
Self { equality: true, comparison: false }
}
}
pub fn matching_field_ids(

2
crates/milli/src/index.rs
View File

@ -896,7 +896,7 @@ impl Index {
let mut fields_ids = HashSet::new();
for (field_id, field_name) in fields_ids_map.iter() {
if match_pattern_by_features(field_name, &filterable_fields, &|features| {
features.is_filterable_order()
features.is_filterable_comparison()
}) == PatternMatch::Match
{
fields_ids.insert(field_id);

2
crates/milli/src/lib.rs
View File

@ -54,6 +54,7 @@ pub use {charabia as tokenizer, heed, rhai};
pub use self::asc_desc::{AscDesc, AscDescError, Member, SortError};
pub use self::attribute_patterns::AttributePatterns;
pub use self::attribute_patterns::PatternMatch;
pub use self::criterion::{default_criteria, Criterion, CriterionError};
pub use self::error::{
Error, FieldIdMapMissingEntry, InternalError, SerializationError, UserError,
@ -72,7 +73,6 @@ pub use self::heed_codec::{
};
pub use self::index::Index;
pub use self::localized_attributes_rules::LocalizedAttributesRule;
use self::localized_attributes_rules::LocalizedFieldIds;
pub use self::search::facet::{FacetValueHit, SearchForFacetValues};
pub use self::search::similar::Similar;
pub use self::search::{

37
crates/milli/src/search/facet/filter.rs
View File

@ -273,22 +273,21 @@ impl<'a> Filter<'a> {
| Condition::LowerThan(_)
| Condition::LowerThanOrEqual(_)
| Condition::Between { .. }
if !features.is_filterable_order() =>
if !features.is_filterable_comparison() =>
{
/// TODO produce an dedicated error for this
todo!("filtering on non-ordered fields is not supported, return an error")
return Err(generate_filter_error(rtxn, index, field_id, operator, features));
}
Condition::Empty if !features.is_filterable_empty() => {
/// TODO produce an dedicated error for this
todo!("filtering on non-empty fields is not supported, return an error")
return Err(generate_filter_error(rtxn, index, field_id, operator, features));
}
Condition::Null if !features.is_filterable_null() => {
/// TODO produce an dedicated error for this
todo!("filtering on non-null fields is not supported, return an error")
return Err(generate_filter_error(rtxn, index, field_id, operator, features));
}
Condition::Exists if !features.is_filterable_exists() => {
/// TODO produce an dedicated error for this
todo!("filtering on non-exists fields is not supported, return an error")
return Err(generate_filter_error(rtxn, index, field_id, operator, features));
}
Condition::Equal(_) | Condition::NotEqual(_) if !features.is_filterable_equality() => {
return Err(generate_filter_error(rtxn, index, field_id, operator, features));
}
Condition::GreaterThan(val) => {
(Excluded(val.parse_finite_float()?), Included(f64::MAX))
@ -737,6 +736,26 @@ impl<'a> Filter<'a> {
}
}
fn generate_filter_error(
rtxn: &heed::RoTxn<'_>,
index: &Index,
field_id: FieldId,
operator: &Condition<'_>,
features: &FilterableAttributesFeatures,
) -> Error {
match index.fields_ids_map(rtxn) {
Ok(fields_ids_map) => {
let field = fields_ids_map.name(field_id).unwrap_or_default();
Error::UserError(UserError::FilterOperatorNotAllowed {
field: field.to_string(),
allowed_operators: features.allowed_filter_operators(),
operator: operator.operator().to_string(),
})
}
Err(e) => e.into(),
}
}
impl<'a> From<FilterCondition<'a>> for Filter<'a> {
fn from(fc: FilterCondition<'a>) -> Self {
Self { condition: fc }

7
crates/milli/src/update/facet/bulk.rs
View File

@ -365,8 +365,6 @@ impl<R: std::io::Read + std::io::Seek> FacetsUpdateBulkInner<R> {
mod tests {
use std::iter::once;
use big_s::S;
use maplit::hashset;
use roaring::RoaringBitmap;
use crate::documents::mmap_from_objects;
@ -474,9 +472,8 @@ mod tests {
index
.update_settings(|settings| {
settings.set_primary_key("id".to_owned());
settings.set_filterable_fields(vec![FilterableAttributesRule::Field(
"id".to_string(),
)]);
settings
.set_filterable_fields(vec![FilterableAttributesRule::Field("id".to_string())]);
})
.unwrap();

17
crates/milli/src/update/new/extract/mod.rs
View File

@ -5,7 +5,6 @@ mod geo;
mod searchable;
mod vectors;
use bumpalo::Bump;
pub use cache::{
merge_caches_sorted, transpose_and_freeze_caches, BalancedCaches, DelAddRoaringBitmap,
};
@ -15,22 +14,6 @@ pub use geo::*;
pub use searchable::*;
pub use vectors::EmbeddingExtractor;
use super::indexer::document_changes::{DocumentChanges, IndexingContext};
use super::steps::IndexingStep;
use super::thread_local::{FullySend, ThreadLocal};
use crate::Result;
pub trait DocidsExtractor {
fn run_extraction<'pl, 'fid, 'indexer, 'index, 'extractor, DC: DocumentChanges<'pl>, MSP>(
document_changes: &DC,
indexing_context: IndexingContext<'fid, 'indexer, 'index, MSP>,
extractor_allocs: &'extractor mut ThreadLocal<FullySend<Bump>>,
step: IndexingStep,
) -> Result<Vec<BalancedCaches<'extractor>>>
where
MSP: Fn() -> bool + Sync;
}
/// TODO move in permissive json pointer
pub mod perm_json_p {
use serde_json::{Map, Value};

14
crates/milli/src/update/new/extract/searchable/extract_word_docids.rs
View File

@ -5,7 +5,6 @@ use std::ops::DerefMut as _;
use bumpalo::collections::vec::Vec as BumpVec;
use bumpalo::Bump;
use heed::RoTxn;
use super::match_searchable_field;
use super::tokenize_document::{tokenizer_builder, DocumentTokenizer};
@ -18,7 +17,7 @@ use crate::update::new::ref_cell_ext::RefCellExt as _;
use crate::update::new::steps::IndexingStep;
use crate::update::new::thread_local::{FullySend, MostlySend, ThreadLocal};
use crate::update::new::DocumentChange;
use crate::{bucketed_position, DocumentId, FieldId, Index, Result, MAX_POSITION_PER_ATTRIBUTE};
use crate::{bucketed_position, DocumentId, FieldId, Result, MAX_POSITION_PER_ATTRIBUTE};
const MAX_COUNTED_WORDS: usize = 30;
@ -407,15 +406,4 @@ impl WordDocidsExtractors {
let mut buffer = BumpVec::with_capacity_in(buffer_size, &context.doc_alloc);
cached_sorter.flush_fid_word_count(&mut buffer)
}
fn attributes_to_extract<'a>(
rtxn: &'a RoTxn,
index: &'a Index,
) -> Result<Option<Vec<&'a str>>> {
index.user_defined_searchable_fields(rtxn).map_err(Into::into)
}
fn attributes_to_skip<'a>(_rtxn: &'a RoTxn, _index: &'a Index) -> Result<Vec<&'a str>> {
Ok(Vec::new())
}
}

21
crates/milli/src/update/new/indexer/post_processing.rs
View File

@ -33,10 +33,8 @@ where
{
let index = indexing_context.index;
indexing_context.progress.update_progress(IndexingStep::PostProcessingFacets);
if index.facet_search(wtxn)? {
compute_facet_search_database(index, wtxn, global_fields_ids_map)?;
}
compute_facet_level_database(index, wtxn, facet_field_ids_delta)?;
compute_facet_level_database(index, wtxn, facet_field_ids_delta, &global_fields_ids_map)?;
compute_facet_search_database(index, wtxn, global_fields_ids_map)?;
indexing_context.progress.update_progress(IndexingStep::PostProcessingWords);
if let Some(prefix_delta) = compute_word_fst(index, wtxn)? {
compute_prefix_database(index, wtxn, prefix_delta, indexing_context.grenad_parameters)?;
@ -116,6 +114,12 @@ fn compute_facet_search_database(
global_fields_ids_map: GlobalFieldsIdsMap,
) -> Result<()> {
let rtxn = index.read_txn()?;
// if the facet search is not enabled, we can skip the rest of the function
if !index.facet_search(wtxn)? {
return Ok(());
}
let localized_attributes_rules = index.localized_attributes_rules(&rtxn)?;
let filterable_attributes_rules = index.filterable_attributes_rules(&rtxn)?;
let mut facet_search_builder = FacetSearchBuilder::new(
@ -166,11 +170,16 @@ fn compute_facet_level_database(
index: &Index,
wtxn: &mut RwTxn,
mut facet_field_ids_delta: FacetFieldIdsDelta,
global_fields_ids_map: &GlobalFieldsIdsMap,
) -> Result<()> {
let facet_leveled_field_ids = index.facet_leveled_field_ids(&*wtxn)?;
let rtxn = index.read_txn()?;
let filterable_attributes_rules = index.filterable_attributes_rules(&rtxn)?;
for (fid, delta) in facet_field_ids_delta.consume_facet_string_delta() {
// skip field ids that should not be facet leveled
if !facet_leveled_field_ids.contains(&fid) {
let Some(metadata) = global_fields_ids_map.metadata(fid) else {
continue;
};
if !metadata.require_facet_level_database(&filterable_attributes_rules) {
continue;
}

6
crates/milli/tests/search/facet_distribution.rs
View File

@ -6,7 +6,7 @@ use milli::progress::Progress;
use milli::update::new::indexer;
use milli::update::{IndexDocumentsMethod, IndexerConfig, Settings};
use milli::vector::EmbeddingConfigs;
use milli::{FacetDistribution, FilterableAttributesSettings, Index, Object, OrderBy};
use milli::{FacetDistribution, FilterableAttributesRule, Index, Object, OrderBy};
use serde_json::{from_value, json};
#[test]
@ -21,8 +21,8 @@ fn test_facet_distribution_with_no_facet_values() {
let mut builder = Settings::new(&mut wtxn, &index, &config);
builder.set_filterable_fields(vec![
FilterableAttributesSettings::Field(S("genres")),
FilterableAttributesSettings::Field(S("tags")),
FilterableAttributesRule::Field(S("genres")),
FilterableAttributesRule::Field(S("tags")),
]);
builder.execute(|_| (), || false).unwrap();
wtxn.commit().unwrap();

15
crates/milli/tests/search/mod.rs
View File

@ -12,8 +12,7 @@ use milli::update::new::indexer;
use milli::update::{IndexDocumentsMethod, IndexerConfig, Settings};
use milli::vector::EmbeddingConfigs;
use milli::{
AscDesc, Criterion, DocumentId, FilterableAttributesSettings, Index, Member,
TermsMatchingStrategy,
AscDesc, Criterion, DocumentId, FilterableAttributesRule, Index, Member, TermsMatchingStrategy,
};
use serde::{Deserialize, Deserializer};
use slice_group_by::GroupBy;
@ -46,12 +45,12 @@ pub fn setup_search_index_with_criteria(criteria: &[Criterion]) -> Index {
builder.set_criteria(criteria.to_vec());
builder.set_filterable_fields(vec![
FilterableAttributesSettings::Field(S("tag")),
FilterableAttributesSettings::Field(S("asc_desc_rank")),
FilterableAttributesSettings::Field(S("_geo")),
FilterableAttributesSettings::Field(S("opt1")),
FilterableAttributesSettings::Field(S("opt1.opt2")),
FilterableAttributesSettings::Field(S("tag_in")),
FilterableAttributesRule::Field(S("tag")),
FilterableAttributesRule::Field(S("asc_desc_rank")),
FilterableAttributesRule::Field(S("_geo")),
FilterableAttributesRule::Field(S("opt1")),
FilterableAttributesRule::Field(S("opt1.opt2")),
FilterableAttributesRule::Field(S("tag_in")),
]);
builder.set_sortable_fields(hashset! {
S("tag"),