本番環境対応、10規模のベクトル検索 - Elasticsearch

POST _inference/my-e5-endpoint
{
  "input": "How many adult mallard ducks fit in an american football field?"
}

PUT my-index
{
  "mappings": {
    "properties": {
      "my_vector": {
        "type": "dense_vector"
      },
      "my_text" : {
        "type" : "text"
      },
      "category": {
        "type": "text"
      }
    }
  }
}



# Index a document
PUT my-index/_doc/2
{
  "my_vector" : [
        0.018743202,
        -0.011059246,
   …
        0.05155631,
        0.04044561
      ],
  "my_text" :"57,000 average sized mallard ducks fit on an American football field",
"category" : "weird_stats"
}

GET /my-index/_search
{
  "retriever": {
    "knn": { 
      "field": "my_vector", 
      "query_vector_builder":{
      "text_embedding": {
        "model_id": "my-e5-endpoint",
        "model_text": "How many adult mallard ducks fit in an american football field??"
      }
    }, 
      "k": 10, 
      "num_candidates": 10 
    }
  }
}

GET /my-index/_search
{
  "retriever": {
    "knn": {
      "field": "my_vector",
      "query_vector_builder":{
      "text_embedding": {
        "model_id": "my-e5-endpoint",
        "model_text": "How many adult mallard ducks fit in an american football field??"
      }
    }, 
      "k": 10,
      "num_candidates": 10,
      "filter": [
        {
          "term": {
            "category": "weird_stats"
          }
        }
      ]
    }
  }
}

GET /my-index/_search
{
  "retriever": {
    "rrf": {
      "retrievers": [
        {
          "standard": {
            "query": {
              "bool": {
                "filter": [
                  {
                    "term": {
                      "category": "weird_stats"
                    }
                  }
                ]
              }
            }
          }
        },
        {
          "knn": {
            "field": "vector",
            "query_vector_builder": {
              "text_embedding": {
                "model_id": "my-e5-endpoint",
                "model_text": "How many adult mallard ducks fit in an american football field??"
              }
            },
            "k": 10,
            "num_candidates": 10
          }
        }
      ],
      "rank_constant": 1,
      "rank_window_size": 50
    }
  }
}

POST /_security/role/employee_role {
  "indices": [
    {
      "names": [ "example-index" ],
      "privileges": [ "read" ],
      "field_security" : {
        "grant" : [ "state", "name.first", "name.last" ]
      }
    }
  ]
}


GET example-index/_search {
  "query": {...},
  "knn" = {...},
  "aggs": {...}
}