from fashion_clip.fashion_clip import FashionCLIP from PIL import Image import faiss import numpy as np MODEL_NAME = "patrickjohncyh/fashion-clip" # FashionCLIP 2.0 QUERY_IMAGE = "image03/00.png" TOP_K = 100 IMAGE_PATHS = [ "image03/00.png", "image03/01.png", "image03/02.png", "image03/03.png", "image03/04.png", "image03/05.png", "image03/06.png", "image03/07.png", "image03/08.png", "image03/09.png", ] # 최종 점수 비율 WEIGHTS = { "clip": 0.35, "color": 0.20, "brightness": 0.15, "texture": 0.15, "shape": 0.15, } # 같은 옷 / 비슷한 옷 판단 기준 THRESHOLDS = { "same": 0.80, "similar": 0.70, "min_color": 0.60, "min_brightness": 0.60, "min_texture": 0.60, "min_shape": 0.60, } def load_image(path): return Image.open(path).convert("RGB") def get_cloth_mask(image): # 흰 배경을 제외한 옷 영역 rgb = np.asarray(image) white = (rgb[:, :, 0] > 245) & (rgb[:, :, 1] > 245) & (rgb[:, :, 2] > 245) mask = ~white if not mask.any(): mask = np.ones(mask.shape, dtype=bool) return mask def normalize(vector): vector = vector.astype("float32").ravel() norm = np.linalg.norm(vector) return vector / norm if norm > 0 else vector def normalize_rows(matrix): matrix = matrix.astype("float32") norm = np.linalg.norm(matrix, axis=1, keepdims=True) norm[norm == 0] = 1 return np.ascontiguousarray(matrix / norm) def color_feature(image, mask): # 색상 비교 hsv = np.asarray(image.convert("HSV")) hist, _ = np.histogramdd( hsv[mask], bins=(12, 6, 6), range=((0, 255), (0, 255), (0, 255)), ) return normalize(hist) def brightness_feature(image, mask): # 밝기 비교: 연청/진청/흑청 구분에 도움 hsv = np.asarray(image.convert("HSV")) brightness = hsv[:, :, 2][mask] / 255.0 hist, _ = np.histogram(brightness, bins=16, range=(0, 1)) hist = hist.astype("float32") if hist.sum() > 0: hist /= hist.sum() return { "hist": hist, "mean": float(brightness.mean()), } def texture_feature(image, mask): # 질감/워싱 비교: 밝기 변화량을 히스토그램으로 저장 gray = np.asarray(image.convert("L"), dtype=np.float32) / 255.0 edge = np.zeros_like(gray) edge[:, 1:] += np.abs(np.diff(gray, axis=1)) edge[1:, :] += np.abs(np.diff(gray, axis=0)) hist, _ = np.histogram(edge[mask], bins=16, range=(0, 1)) return normalize(hist) def shape_feature(mask): # 형태 비교: 가로세로 비율 + 32x32 실루엣 y, x = np.where(mask) crop = mask[y.min():y.max() + 1, x.min():x.max() + 1] height, width = crop.shape resample = Image.Resampling.BILINEAR if hasattr(Image, "Resampling") else Image.BILINEAR silhouette = Image.fromarray((crop * 255).astype("uint8")).resize((32, 32), resample) silhouette = np.asarray(silhouette, dtype=np.float32) / 255.0 return { "aspect": width / max(height, 1), "silhouette": silhouette, } def extract_features(image): mask = get_cloth_mask(image) return { "color": color_feature(image, mask), "brightness": brightness_feature(image, mask), "texture": texture_feature(image, mask), "shape": shape_feature(mask), } def histogram_score(a, b): return float(np.dot(a, b)) def brightness_score(a, b): hist_score = float(np.minimum(a["hist"], b["hist"]).sum()) mean_score = max(0.0, 1.0 - abs(a["mean"] - b["mean"]) / 0.5) return (hist_score * 0.7) + (mean_score * 0.3) def shape_score(a, b): aspect_diff = abs(np.log(a["aspect"] + 1e-6) - np.log(b["aspect"] + 1e-6)) aspect_score = max(0.0, 1.0 - aspect_diff / 0.5) intersection = np.minimum(a["silhouette"], b["silhouette"]).sum() union = np.maximum(a["silhouette"], b["silhouette"]).sum() silhouette_score = float(intersection / union) if union > 0 else 0.0 return (aspect_score * 0.4) + (silhouette_score * 0.6) def search_with_faiss(fclip, images, query_image): # FashionCLIP 이미지 임베딩으로 FAISS 상위 TOP_K개 검색 image_vectors = normalize_rows(fclip.encode_images(images, batch_size=len(images))) query_vector = normalize_rows(fclip.encode_images([query_image], batch_size=1)) index = faiss.IndexFlatIP(image_vectors.shape[1]) index.add(image_vectors) scores, indices = index.search(query_vector, min(TOP_K, len(images))) return [ (int(idx), float(score)) for score, idx in zip(scores[0], indices[0]) ] def compare(query_features, candidate_features, clip_score): scores = { "clip": clip_score, "color": histogram_score(query_features["color"], candidate_features["color"]), "brightness": brightness_score( query_features["brightness"], candidate_features["brightness"], ), "texture": histogram_score(query_features["texture"], candidate_features["texture"]), "shape": shape_score(query_features["shape"], candidate_features["shape"]), } scores["final"] = sum(scores[name] * WEIGHTS[name] for name in WEIGHTS) return scores def classify(scores): if ( scores["final"] >= THRESHOLDS["same"] and scores["color"] >= THRESHOLDS["min_color"] and scores["brightness"] >= THRESHOLDS["min_brightness"] and scores["texture"] >= THRESHOLDS["min_texture"] and scores["shape"] >= THRESHOLDS["min_shape"] ): return "same" if scores["final"] >= THRESHOLDS["similar"]: return "similar" return "different" def print_result(path, scores): print( "matched image:", path, "match:", classify(scores), "final_score:", f"{scores['final']:.3f}", "clip_score:", f"{scores['clip']:.3f}", "color_score:", f"{scores['color']:.3f}", "brightness_score:", f"{scores['brightness']:.3f}", "texture_score:", f"{scores['texture']:.3f}", "shape_score:", f"{scores['shape']:.3f}", ) fclip = FashionCLIP(MODEL_NAME) # 1. 이미지 로드 images = [load_image(path) for path in IMAGE_PATHS] query_image = load_image(QUERY_IMAGE) # 2. 색상/밝기/질감/형태 특징 추출 image_features = [extract_features(image) for image in images] query_features = extract_features(query_image) # 3. FAISS로 clip_score 상위 5개 검색 faiss_results = search_with_faiss(fclip, images, query_image) # 4. 상위 5개만 파일명 순서로 출력 for idx, clip_score in sorted(faiss_results): scores = compare(query_features, image_features[idx], clip_score) print_result(IMAGE_PATHS[idx], scores) # 1. Marqo/marqo-fashionSigLIP 가장 추천 # 2. Marqo/marqo-fashionCLIP 그다음 추천 # 3. patrickjohncyh/fashion-clip 현재 사용 중인 FashionCLIP 2.0 # 이유는 Marqo 쪽 모델들이 FashionCLIP 2.0보다 패션 검색 벤치마크에서 더 좋은 결과를 냈다고 공개되어 있습니다. 특히 Marqo/marqo-fashionSigLIP가 가장 높습니다. # Marqo/marqo-fashionSigLIP: 성능 가장 좋음, 모델 크기 약 0.2B # Marqo/marqo-fashionCLIP: FashionCLIP 2.0보다 좋고, 크기 약 0.1B라 가벼움 # patrickjohncyh/fashion-clip: 안정적이고 현재 코드와 호환 쉬움