hackathon-tenderhack/machine-learning/image_hashing.py at main · one-zero-eight/hackathon-tenderhack · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
from collections import defaultdict
from io import BytesIO
from pathlib import Path
from traceback import print_exc

import imagehash
import openpyxl
import pandas as pd
import requests
from joblib import Memory, Parallel, delayed
from PIL import Image
from tqdm import tqdm

memory = Memory(location=".cache", verbose=0)

DATA_PATH = Path("Исходные данные_Хакатон_Казань_20251128_1800.xlsx")
dfs = pd.read_excel(DATA_PATH, sheet_name=None)
list1 = dfs["Лист1"]
list2 = dfs["Лист2"]


@memory.cache
def get_image_hash_from_url(url: str) -> str | None:
    try:
        response = requests.get(url)
        image = Image.open(BytesIO(response.content))
        if image.mode == "P":  # palette mode with possible transparency
            image = image.convert("RGBA")
        hash_as_array = imagehash.phash(image)
        hash_as_str = str(hash_as_array)
        return hash_as_str
    except Exception:
        print_exc()
        return None


list1_urls = list1["ссылка на картинку сте"].dropna().unique()
list2_url = list2["ссылка на картинку сте"].dropna().unique()

list1_hashes = Parallel(n_jobs=-1)(
    delayed(get_image_hash_from_url)(url) for url in tqdm(list1_urls, desc="Processing list1")
)
list2_hashes = Parallel(n_jobs=-1)(
    delayed(get_image_hash_from_url)(url) for url in tqdm(list2_url, desc="Processing list2")
)

# Create URL to hash mappings
list1_url_to_hash = dict(zip(list1_urls, list1_hashes))
list2_url_to_hash = dict(zip(list2_url, list2_hashes))

# Set hash column based on URL match
list1["hash"] = list1["ссылка на картинку сте"].map(list1_url_to_hash)
list2["hash"] = list2["ссылка на картинку сте"].map(list2_url_to_hash)

clusters1 = defaultdict(list)
for url, hash in list1_url_to_hash.items():
    clusters1[hash].append(url)

clusters2 = defaultdict(list)
for url, hash in list2_url_to_hash.items():
    clusters2[hash].append(url)

print(f"Clusters1: {len(clusters1)}")
print(f"Clusters2: {len(clusters2)}")

# Now for each row in excel add hash to the column J (url column is C), right in xlsx file via openpyxl
workbook = openpyxl.load_workbook(DATA_PATH)
sheet1 = workbook["Лист1"]
sheet1.cell(row=1, column=10, value="hash")
for i, row in tqdm(
    enumerate(sheet1.iter_rows(min_row=2, max_row=sheet1.max_row, values_only=True)), desc="Adding hashes"
):
    url = row[2]
    hash = list1_url_to_hash[url]
    sheet1.cell(row=i + 2, column=10, value=hash)

sheet2 = workbook["Лист2"]
sheet2.cell(row=1, column=10, value="hash")
for i, row in tqdm(
    enumerate(sheet2.iter_rows(min_row=2, max_row=sheet2.max_row, values_only=True)), desc="Adding hashes"
):
    url = row[2]
    hash = list2_url_to_hash[url]
    sheet2.cell(row=i + 2, column=10, value=hash)

workbook.save(DATA_PATH.with_name(DATA_PATH.stem + "_hashed2.xlsx"))