Elasticsearch と SigLIP-2 による山頂のマルチモーダル探索

def l2norm(v: np.ndarray) -> np.ndarray:
    return v / (np.linalg.norm(v) + 1e-12)
def compute_blended_peak_vec(
        emb: Siglip2,
        names: List[str],
        peak_id: str,
        peaks_images_root: str,
        alpha_text: float = 0.5,
        max_images: int = 3,
) -> Tuple[np.ndarray, int, int, List[str]]:
    """
    Build blended vector for a single peak.

    Returns:
      vec           : np.ndarray (L2-normalized)
      found_count   : number of reference images discovered
      used_count    : number of references used (<= max_images)
      used_filenames: list of filenames used (for logging)
    """
    # 1) TEXT vector
    tv = embed_text_blend(emb, names)

    # 2) IMAGE refs: prefer folder by id; fallback to slug of the primary name
    root = Path(peaks_images_root)
    candidates = [root / peak_id]
    if names:
        candidates.append(root / slugify(names[0]))

    all_refs: List[Path] = []
    for c in candidates:
        if c.exists() and c.is_dir():
            all_refs = list_ref_images(c)
            if all_refs:
                break

    found = len(all_refs)
    used_list = all_refs[:max_images] if (max_images and found > max_images) else all_refs
    used = len(used_list)

    img_v = embed_image_mean(emb, used_list) if used_list else None

    # 3) Blend TEXT and IMAGE vectors, clamp alpha to [0,1]
    a = max(0.0, min(1.0, float(alpha_text)))
    vec = l2norm(tv if img_v is None else (a * tv + (1.0 - a) * img_v)).astype("float32")
    return vec, found, used, [p.name for p in used_list]

def bulk_index_photos(
        es: Elasticsearch,
        images_root: str,
        photos_index: str = "photos",
        peaks_index: str = "peaks_catalog",
        topk_predicted: int = 5,
        batch_size: int = 200,
        refresh: str = "false",
) -> None:
    """Walk a folder of images, embed + enrich, and bulk index to Elasticsearch."""
    root = Path(images_root)
    if not root.exists():
        raise SystemExit(f"Images root not found: {images_root}")

    emb = Siglip2()
    batch: List[Dict[str, Any]] = []
    n_indexed = 0

    for p in iter_images(root):
        rel = relpath_within(root, p)
        _id = id_for_path(rel)

        # 1) Image embedding (and reuse it for predicted_peaks)
        try:
            with Image.open(p) as im:
                ivec = emb.image_vec(im.convert("RGB")).astype("float32")
        except (UnidentifiedImageError, OSError) as e:
            print(f"[skip] {rel} — cannot embed: {e}")
            continue

        # 2) Predict top-k peak names
        try:
            top_names = predict_peaks(es, ivec.tolist(), peaks_index=peaks_index, k=topk_predicted)
        except Exception as e:
            print(f"[warn] predict_peaks failed for {rel}: {e}")
            top_names = []

        # 3) EXIF enrichment (safe)
        gps = get_gps_decimal(str(p))
        shot = get_shot_time(str(p))

        # 4) Build doc and stage for bulk
        doc = {"path": rel, "clip_image": ivec.tolist(), "predicted_peaks": top_names}
        if gps:
            doc["gps"] = gps
        if shot:
            doc["shot_time"] = shot

        batch.append(
            {"_op_type": "index", "_index": photos_index, "_id": _id, "_source": doc}
        )

        # 5) Periodic flush
        if len(batch) >= batch_size:
            helpers.bulk(es, batch, refresh=refresh)
            n_indexed += len(batch)
            print(f"[photos] indexed {n_indexed} (last: {rel})")
            batch.clear()

    # Final flush
    if batch:
        helpers.bulk(es, batch, refresh=refresh)
        n_indexed += len(batch)
        print(f"[photos] indexed {n_indexed} total.")

    print("[done] photos indexing")

POST photos/_search
{
  "knn": {
    "field": "clip_image",
    "query_vector": [ ... ],
    "k": 60,
    "num_candidates": 2000
  },
  "query": {
    "bool": {
      "filter": [
        { "geo_bounding_box": { "gps": { "top_left": "...", "bottom_right": "..." } } }
      ]
    }
  },
  "_source": ["path","predicted_peaks","gps","shot_time"]
}

Response (first two documents):
{
 "hits": {
   "total": {
     "value": 56,
     "relation": "eq"
   },
   "max_score": 0.5779596,
   "hits": [
     {
       "_index": "photos",
       "_id": "d01da3a1141981486c3493f6053c79e92a788463",
       "_score": 0.5779596,
       "_source": {
         "path": "IMG_2738.HEIC",
         "predicted_peaks": [
           "Pumori",
           "Kyajo Ri",
           "Khumbila",
           "Nangkartshang",
           "Kongde Ri"
         ],
         "gps": {
           "lat": 27.97116388888889,
           "lon": 86.82331111111111
         },
         "shot_time": "2023-11-03T08:07:13"
       }
     },
     {
       "_index": "photos",
       "_id": "c79d251f07adc5efaedc53561110a7fd78e23914",
       "_score": 0.5766071,
       "_source": {
         "path": "IMG_2761.HEIC",
         "predicted_peaks": [
           "Kyajo Ri",
           "Makalu",
           "Baruntse",
           "Cho Oyu",
           "Khumbila"
         ],
         "gps": {
           "lat": 27.975558333333332,
           "lon": 86.82515
         },
         "shot_time": "2023-11-03T08:51:08"
       }
     }
}

GET peaks_catalog/_search
{
 "knn": {
   "field": "text_embed",
   "query_vector": [...image-vector... ],
   "k": 3,
   "num_candidates": 500
 },
 "_source": [
   "id",
   "names",
   "latlon",
   "text_embed"
 ]
}


Response (first two documents):
{
 "took": 2,
 "timed_out": false,
 "_shards": {
   "total": 1,
   "successful": 1,
   "skipped": 0,
   "failed": 0
 },
 "hits": {
   "total": {
     "value": 3,
     "relation": "eq"
   },
   "max_score": 0.58039916,
   "hits": [
     {
       "_index": "peaks_catalog",
       "_id": "pumori",
       "_score": 0.58039916,
       "_source": {
         "id": "pumori",
         "names": [
           "Pumori",
           "Pumo Ri"
         ],
         "latlon": {
           "lat": 28.01472,
           "lon": 86.82806
         },
         "text_embed": [
                  ... embeddings...
         ]
       }
     },
     {
       "_index": "peaks_catalog",
       "_id": "kyajo-ri",
       "_score": 0.57942784,
       "_source": {
         "id": "kyajo-ri",
         "names": [
           "Kyajo Ri",
           "Kyazo Ri"
         ],
         "latlon": {
           "lat": 27.909167,
           "lon": 86.673611
         },
         "text_embed": [
           ... embeddings...
         ]
       }
     }
   ]
 }
}

POST photos/_search
{
 "knn": {
   "field": "clip_image",
   "query_vector": [ ...image-vector... ],
   "k": 30,
   "num_candidates": 2000
 },
 "_source": [
   "path",
   "gps",
   "shot_time",
   "predicted_peaks",
   "clip_image"
 ],
 "query": {
   "bool": {
     "filter": [
       {
         "term": {
           "predicted_peaks": "Pumori"
         }
       }
     ]
   }
 }
}


Response (first two documents):
{
 "hits": {
   "total": {
     "value": 56,
     "relation": "eq"
   },
   "max_score": 0.5779596,
   "hits": [
     {
       "_index": "photos",
       "_id": "d01da3a1141981486c3493f6053c79e92a788463",
       "_score": 0.5779596,
       "_source": {
         "path": "IMG_2738.HEIC",
         "predicted_peaks": [
           "Pumori",
           "Kyajo Ri",
           "Khumbila",
           "Nangkartshang",
           "Kongde Ri"
         ],
         "gps": {
           "lat": 27.97116388888889,
           "lon": 86.82331111111111
         },
         "shot_time": "2023-11-03T08:07:13"
       }
     },
     {
       "_index": "photos",
       "_id": "c79d251f07adc5efaedc53561110a7fd78e23914",
       "_score": 0.5766071,
       "_source": {
         "path": "IMG_2761.HEIC",
         "predicted_peaks": [
           "Kyajo Ri",
           "Makalu",
           "Baruntse",
           "Cho Oyu",
           "Khumbila"
         ],
         "gps": {
           "lat": 27.975558333333332,
           "lon": 86.82515
         },
         "shot_time": "2023-11-03T08:51:08"
       }
     }
}