在 Elasticsearch 中部署多语言嵌入模型

import requests
import json
import os
### Download multilingual coco dataset into a json file (for easy viewing)
### Here we are retrieving first 100 rows for this example
### Alternatively, you can use `datasets` library from Hugging Face
url = "https://datasets-server.huggingface.co/rows?dataset=romrawinjp%2Fmultilingual-coco&config=default&split=restval&offset=0&length=100"
response = requests.get(url)


if response.status_code == 200:
   data = response.json()
   output_file = "multilingual_coco_sample.json" 
   ### Loading the downloaded content into a json file locally
   with open(output_file, "w", encoding="utf-8") as f:
       json.dump(data, f, indent=4, ensure_ascii=False)
   print(f"Data successfully downloaded and saved to {output_file}")
else:
   print(f"Failed to download data: {response.status_code}")
   print(response.text)

from elasticsearch import Elasticsearch
from getpass import getpass


# Initialize Elasticsearch client
es = Elasticsearch(getpass("Host: "), api_key=getpass("API Key: "))


index_name = "coco"


# Create the index if it doesn't exist
if not es.indices.exists(index=index_name):
   es.indices.create(index=index_name, body=mapping)

# Load the JSON data
with open('./multilingual_coco_sample.json', 'r') as f:
   data = json.load(f)


rows = data["rows"]
# List of languages to process
languages = ["en", "es", "de", "it", "vi", "th"]


# For each image, we will process each individual caption as its own document
bulk_data = []
for data in rows:
   row = data["row"]
   image = row.get("image")
   image_url = image["src"]


   # Process each language
   for lang in languages:
       # Skip if language not present in this row
       if lang not in row:
           continue


       # Get all descriptions for this language
 # along with first available English caption for reference
       descriptions = row[lang]
       first_eng_caption = row["en"][0]


       # Prepare bulk indexing data
       for description in descriptions:
           if description == "":
               continue
           # Add index operation
           bulk_data.append(
               {"index": {"_index": index_name}}
           )
           # Add document
           bulk_data.append({
               "language": lang,
               "description": description,
               "en": first_eng_caption,
               "image_url": image_url,
           })


# Perform bulk indexing
if bulk_data:
   try:
       response = es.bulk(operations=bulk_data)
       if response["errors"]:
           print("Some documents failed to index")
       else:
           print(f"Successfully bulk indexed {len(bulk_data)} documents")
   except Exception as e:
       print(f"Error during bulk indexing: {str(e)}")


print("Indexing complete!")

export MODEL_ID="bge-m3"
export HUB_MODEL_ID="BAAI/bge-m3"
export CLOUD_ID={{CLOUD_ID}}
export ES_API_KEY={{API_KEY}}
docker run -it --rm docker.elastic.co/eland/eland \
eland_import_hub_model --cloud-id $CLOUD_ID --es-api-key $ES_API_KEY --hub-model-id $HUB_MODEL_ID --es-model-id $MODEL_ID --task-type text_embedding --start

PUT _ingest/pipeline/vectorize_descriptions
{
"description": "Pipeline to run the descriptions text_field through our inference text embedding model",
"processors": [
 {
   "set": {
     "field": "temp_desc",
     "value": "passage: {{description}}"
   }
 },
 {
   "inference": {     
"field_map": {
       "temp_desc": "text_field"
     },
     "model_id": ".multilingual-e5-small_linux-x86_64_search",
     "target_field": "vector_description"
   }
 },
 {
   "remove": {
     "field": "temp_desc"
   }
 }
]
}

PUT coco_multi
{
 "mappings": {
   "properties": {
     "description": {
       "type": "text"
     },
     "en": {
       "type": "text"
     },
     "image_url": {
       "type": "keyword"
     },
     "language": {
       "type": "keyword"
     },
     "vector_description.predicted_value": {
       "type": "dense_vector",
       "dims": 384,
       "index": "true",
       "similarity": "cosine",
       "index_options": {
         "type": "bbq_hnsw" 
       }
     }
   }
 }
}

POST _reindex?wait_for_completion=false
{
 "source": {
   "index": "coco"
 },
 "dest": {
   "index": "coco_multilingual",
   "pipeline": "vectorize_descriptions"
 }
}