From 7da73b44dd04ab9e9765c2d27fed6a61589ef8e3 Mon Sep 17 00:00:00 2001 From: Xiao-congxi <57519220+Xiao-congxi@users.noreply.github.com> Date: Thu, 5 Sep 2024 14:26:09 +0800 Subject: [PATCH] Add files via upload --- research/ReFound/README.md | 121 ++++ research/ReFound/code/MoGETransformer.py | 174 ++++++ research/ReFound/code/configuration.py | 28 + research/ReFound/code/dataset_feature.py | 179 ++++++ research/ReFound/code/feature_based_cap.py | 242 +++++++ research/ReFound/code/feature_based_pop.py | 242 +++++++ research/ReFound/code/feature_based_uvd.py | 275 ++++++++ research/ReFound/code/feature_extraction.sh | 14 + research/ReFound/code/feature_extractor.py | 130 ++++ research/ReFound/code/finetune_cap.py | 256 ++++++++ research/ReFound/code/finetune_pop.py | 256 ++++++++ research/ReFound/code/finetune_uvd.py | 296 +++++++++ research/ReFound/code/ft_dataset_cap.py | 193 ++++++ research/ReFound/code/ft_dataset_pop.py | 193 ++++++ research/ReFound/code/ft_dataset_uvd.py | 190 ++++++ research/ReFound/code/models.py | 588 ++++++++++++++++++ research/ReFound/code/script_feature_based.sh | 67 ++ research/ReFound/code/script_finetune.sh | 67 ++ research/ReFound/code/utils.py | 110 ++++ research/ReFound/requirements.txt | 5 + 20 files changed, 3626 insertions(+) create mode 100644 research/ReFound/README.md create mode 100644 research/ReFound/code/MoGETransformer.py create mode 100644 research/ReFound/code/configuration.py create mode 100644 research/ReFound/code/dataset_feature.py create mode 100644 research/ReFound/code/feature_based_cap.py create mode 100644 research/ReFound/code/feature_based_pop.py create mode 100644 research/ReFound/code/feature_based_uvd.py create mode 100644 research/ReFound/code/feature_extraction.sh create mode 100644 research/ReFound/code/feature_extractor.py create mode 100644 research/ReFound/code/finetune_cap.py create mode 100644 research/ReFound/code/finetune_pop.py create mode 100644 research/ReFound/code/finetune_uvd.py create mode 100644 research/ReFound/code/ft_dataset_cap.py create mode 100644 research/ReFound/code/ft_dataset_pop.py create mode 100644 research/ReFound/code/ft_dataset_uvd.py create mode 100644 research/ReFound/code/models.py create mode 100644 research/ReFound/code/script_feature_based.sh create mode 100644 research/ReFound/code/script_finetune.sh create mode 100644 research/ReFound/code/utils.py create mode 100644 research/ReFound/requirements.txt diff --git a/research/ReFound/README.md b/research/ReFound/README.md new file mode 100644 index 0000000..55615df --- /dev/null +++ b/research/ReFound/README.md @@ -0,0 +1,121 @@ + +## Introduction + +This repo is the [PaddlePaddle](https://www.paddlepaddle.org.cn/en) implementation of the KDD 2024 Reasearch Track paper "ReFound: Crafting a Foundation Model for Urban Region Understanding upon Language and Visual Foundations"([paper link](https://dl.acm.org/doi/pdf/10.1145/3637528.3671992)). + +## Requirement + +* Python >= 3.7 +* paddlepaddle == 2.4.2 + + +## Pre-trained Model +Pretrained model weights of ReFound can be downloaded [here](https://www.dropbox.com/scl/fo/d6rj3r0b2plavmikjsldz/APMM-9LT-DrYx4A_b4scJVk?rlkey=5k5zrfjpxfiu1kuyevmgmvgch&st=f8vkc1xy&dl=0). + + + +## Evaluation Dataset +We provide the processed dataset of two downstream tasks in our paper: Urban Village Detection (UVD) and Population Prediction (POP). +The link is coming soon. + + + + + + + + +## Folder Structure +Please create folders with the following directory structure: +``` +ReFound + |- bert-based-chinese + |- code + |- data + |- checkpoint + |- region_embed + |- log + |- log_feature + |- prob + |- prob_feature +``` +- ./bert-based-chinese/ : download BERT tokenizer +- ./checkpoint/ : the pre-trained ReFound model will be loaded from this dir +- ./region_embed/ : save the features of each region extracted by ReFound model (for feature-based evaluation) +- ./log/ : save log files (for fine-tuning evaluation) +- ./log_feature/ : log files (for feature-based evaluation) +- ./prob/ : model's output probability in UVD binary classification task (for fine-tuning evaluation) +- ./prob_feature/ : model's output probability in UVD binary classification task (for feature-based evaluation) + + +## Usage +The pre-trained ReFound model can be applied to downstream urban region understanding tasks in two ways: *fine-tuning* and *feature-based prediction*. + +### Preparation + +**Step1:** download the evaluation data and put it to ./data/ +**Step2:** download the pre-trained ReFound model and put it to ./checkpoint/ +**Step3:** download Bert tokenizer and put it to ./bert-based-chinese/ + + +### Fine-tuning + +Check hyper-parameters in the file script_finetune.sh, and fine-tune the pre-trained model by: + +``` +# Urban Village Detection (UVD) task +sh script.sh uvd [city] [param1] [param2] ... + +# Popilation Prediction (POP) task +sh script.sh pop [city] [param1] [param2] ... +``` + + +### Feature-based Prediction +Extract the region feature using the pre-trained model by: +``` +sh feature_extraction.sh [city] +``` + + +Check hyper-parameters in the file script_feature_based.sh, and then train the task-specific prediction head: +``` +# Urban Village Detection (UVD) task +sh script_feature_based.sh uvd [city] [param1] [param2] ... + +# Popilation Prediction (POP) task +sh script_feature_based.sh pop [city] [param1] [param2] ... +``` + +### Reference + +If you find this code or any of the ideas in the paper useful, please cite: + +```bibtex +@inproceedings{xiao2024refound, + title={ReFound: Crafting a Foundation Model for Urban Region Understanding upon Language and Visual Foundations}, + author={Xiao, Congxi and Zhou, Jingbo and Xiao, Yixiong and Huang, Jizhou and Xiong, Hui}, + booktitle={Proceedings of the 30th ACM SIGKDD Conference on Knowledge Discovery and Data Mining}, + pages={3527--3538}, + year={2024} +} +``` + + + + + + + diff --git a/research/ReFound/code/MoGETransformer.py b/research/ReFound/code/MoGETransformer.py new file mode 100644 index 0000000..c0936ba --- /dev/null +++ b/research/ReFound/code/MoGETransformer.py @@ -0,0 +1,174 @@ +import math +import paddle +import paddle.nn as nn +import paddle.nn.functional as F + + + +class MoGESelfAttention(nn.Layer): + def __init__(self, config): + super().__init__() + assert config['hidden_size'] % config['num_attention_heads'] == 0 + + self.num_attention_heads = config['num_attention_heads'] + self.attention_head_size = int(config['hidden_size'] / config['num_attention_heads']) + self.all_head_size = self.num_attention_heads * self.attention_head_size + + self.query = nn.Linear(config['hidden_size'], self.all_head_size) + self.key = nn.Linear(config['hidden_size'], self.all_head_size) + self.value = nn.Linear(config['hidden_size'], self.all_head_size) + + self.dropout = nn.Dropout(config['attention_probs_dropout_prob']) + + def transpose_for_scores(self, x): + new_x_shape = tuple(x.shape[:-1]) + (self.num_attention_heads, self.attention_head_size) + x = x.reshape(new_x_shape) + return x.transpose((0, 2, 1, 3)) + + def forward(self, hidden_states, attention_mask): + query_layer = self.transpose_for_scores(self.query(hidden_states)) + key_layer = self.transpose_for_scores(self.key(hidden_states)) + value_layer = self.transpose_for_scores(self.value(hidden_states)) + + attention_scores = paddle.matmul(query_layer, key_layer, transpose_y=True) + attention_scores = attention_scores / math.sqrt(self.attention_head_size) + + if attention_mask is not None: + attention_scores = attention_scores + attention_mask + + attention_probs = nn.functional.softmax(attention_scores, axis=-1) + attention_probs = self.dropout(attention_probs) + + context_layer = paddle.matmul(attention_probs, value_layer) + + context_layer = context_layer.transpose((0, 2, 1, 3)) + new_context_layer_shape = tuple(context_layer.shape[:-2]) + (self.all_head_size,) + context_layer = context_layer.reshape(new_context_layer_shape) + + outputs = context_layer + return outputs + + +class MoGESelfOutput(nn.Layer): + def __init__(self, config): + super().__init__() + self.dense = nn.Linear(config['hidden_size'], config['hidden_size']) + self.LayerNorm = nn.LayerNorm(config['hidden_size'], epsilon=config['layer_norm_eps']) + self.dropout = nn.Dropout(config['hidden_dropout_prob']) + + def forward(self, hidden_states, input_tensor): + hidden_states = self.dense(hidden_states) + hidden_states = self.dropout(hidden_states) + hidden_states = self.LayerNorm(hidden_states + input_tensor) + return hidden_states + + +class MoGEAttention(nn.Layer): + def __init__(self, config): + super().__init__() + self.selfattn = MoGESelfAttention(config) + self.output = MoGESelfOutput(config) + + def forward(self, hidden_states, attention_mask): + selfattn_outputs = self.selfattn(hidden_states, attention_mask) + attention_output = self.output(selfattn_outputs, hidden_states) + outputs = attention_output + return outputs + + +class MoGEIntermediate(nn.Layer): + def __init__(self, config): + super().__init__() + self.dense = nn.Linear(config['hidden_size'], config['intermediate_size']) + self.act_fn = nn.GELU() + + def forward(self, hidden_states): + hidden_states = self.dense(hidden_states) + hidden_states = self.act_fn(hidden_states) + return hidden_states + + +class MoGEOutput(nn.Layer): + def __init__(self, config): + super().__init__() + self.dense = nn.Linear(config['intermediate_size'], config['hidden_size']) + self.LayerNorm = nn.LayerNorm(config['hidden_size'], epsilon=config['layer_norm_eps']) + self.dropout = nn.Dropout(config['hidden_dropout_prob']) + + def forward(self, hidden_states, input_tensor): + hidden_states = self.dense(hidden_states) + hidden_states = self.dropout(hidden_states) + hidden_states = self.LayerNorm(hidden_states + input_tensor) + return hidden_states + + + +class MoGELayer(nn.Layer): + def __init__(self, config, is_ps_expert): + super().__init__() + self.attention = MoGEAttention(config) + + self.poi_intermediate = MoGEIntermediate(config) + self.poi_output = MoGEOutput(config) + + self.img_intermediate = MoGEIntermediate(config) + self.img_output = MoGEOutput(config) + + if is_ps_expert: + self.ps_intermediate = MoGEIntermediate(config) + self.ps_output = MoGEOutput(config) + + + def forward(self, hidden_states, attention_mask, expert_selection, split_idx): + attention_output = self.attention(hidden_states, attention_mask) + + if expert_selection == 'p_and_s': + poi_attention_output = attention_output[:, : split_idx] + img_attention_output = attention_output[:, split_idx :] + + poi_intermediate_output = self.poi_intermediate(poi_attention_output) + poi_mlp_output = self.poi_output(poi_intermediate_output, poi_attention_output) + + img_intermediate_output = self.img_intermediate(img_attention_output) + img_mlp_output = self.img_output(img_intermediate_output, img_attention_output) + + mlp_output = paddle.concat([poi_mlp_output, img_mlp_output], axis=1) + + elif expert_selection == 'ps': + ps_intermediate_output = self.ps_intermediate(attention_output) + ps_mlp_output = self.ps_output(ps_intermediate_output, attention_output) + mlp_output = ps_mlp_output + + return mlp_output + + + + +class MoGEEncoder(nn.Layer): + def __init__(self, config): + super().__init__() + self.config = config + self.ps_layer_start_idx = config['ps_layer_start_idx'] + self.layer = nn.LayerList([MoGELayer(config=config, is_ps_expert=(i >= self.ps_layer_start_idx)) for i in range(config['num_hidden_layers'])]) + + + def forward(self, hidden_states, attention_mask, split_idx): + + for i, layer_module in enumerate(self.layer): + if i < self.ps_layer_start_idx: + hidden_states = layer_module( + hidden_states=hidden_states, + attention_mask=attention_mask, + expert_selection='p_and_s', + split_idx=split_idx, + ) + else: + hidden_states = layer_module( + hidden_states=hidden_states, + attention_mask=attention_mask, + expert_selection='ps', + split_idx=split_idx, + ) + return hidden_states + + \ No newline at end of file diff --git a/research/ReFound/code/configuration.py b/research/ReFound/code/configuration.py new file mode 100644 index 0000000..e47e99e --- /dev/null +++ b/research/ReFound/code/configuration.py @@ -0,0 +1,28 @@ + + +config = { + "attention_probs_dropout_prob": 0.1, + "num_attention_heads": 12, + "num_hidden_layers": 12, + "ps_layer_start_idx": 10, + 'initializer_range': 0.02, + "vocab_size": 21128, + "type_vocab_size": 2, + "max_len_poi": 512, + "max_len_token": 512, + "hidden_size": 768, + "intermediate_size": 3072, + "layer_norm_eps": 1e-12, + "hidden_dropout_prob": 0.1, + "chunk_size_feed_forward": 0, + "poi_cate_num": 130, + "image_size": 256, + "patch_size": 16, + "num_grid_x": 16, + "num_grid_y": 16, + "visual_vocab_size": 8192, + "mask_ratio_poi": 0.15, + "mask_ratio_img": 0.4, + "dvlfm_temp": 0.07, +} + diff --git a/research/ReFound/code/dataset_feature.py b/research/ReFound/code/dataset_feature.py new file mode 100644 index 0000000..384f3d9 --- /dev/null +++ b/research/ReFound/code/dataset_feature.py @@ -0,0 +1,179 @@ +# -*- coding: utf-8 -*- + +import os +import json +import PIL.Image as pil +import random +from transformers import AutoTokenizer +import paddle +from paddle.io import Dataset +from collections import OrderedDict, Counter +from utils import * + + +class Dataset_RegionFeature(Dataset): + def __init__(self, config=None): + super().__init__() + self.config = config + file_path = os.path.abspath(__file__) + self.root_path = os.path.dirname(file_path) + '/../' + self.id_of_region_path = self.root_path + 'data/{c}_id_of_region'.format(c=config['city']) + self.region_coord_path = self.root_path + 'data/{c}_region_coord'.format(c=config['city']) + self.poi_sortby_zorder_path = self.root_path + 'data/{c}_poi_sort_by_zorder'.format(c=config['city']) + self.poi_cate_vocab_path = self.root_path + 'data/poi_cate_vocab' + self.bert_chinese_path = self.root_path + 'bert-base-chinese/' + self.img_path = self.root_path + 'data/satellite_img/{c}/'.format(c=config['city']) + + self.id_of_region = [] + with open(self.id_of_region_path, 'r') as f: + for line in f: + line = line.strip('\n') + self.id_of_region.append(line) + + image_height, image_width = pair(config['image_size']) + patch_height, patch_width = pair(config['patch_size']) + assert image_height % patch_height == 0 and image_width % patch_width == 0, \ + 'Image dimensions must be divisible by the patch size.' + self.num_patch = (image_height // patch_height) * (image_width // patch_width) + + self.max_len_token = config['max_len_token'] + self.max_len_poi = config['max_len_poi'] + self.num_grid_x = config['num_grid_x'] + self.num_grid_y = config['num_grid_y'] + self.num_grid = self.num_grid_x * self.num_grid_y + + + self.poi_zorder_list = [] + with open(self.poi_sortby_zorder_path, 'r') as f: + for line in f: + line = eval(line.strip('\n')) + self.poi_zorder_list.append(line[1]) + + + self.tokenizer = AutoTokenizer.from_pretrained(self.bert_chinese_path) + self.cls_token_id = self.tokenizer.cls_token_id + self.pad_token_id = self.tokenizer.pad_token_id + self.mask_token_id = self.tokenizer.mask_token_id + self.sep_token_id = self.tokenizer.sep_token_id + + + self.region_coord_list = [] + with open(self.region_coord_path, 'r') as f: + for line in f: + line = eval(line.strip('\n')) + self.region_coord_list.append(line) + + self.poi_cate_vocab = {} + with open(self.poi_cate_vocab_path, 'r') as f: + for line in f: + line = line.strip('\n').split('\t') + cate, cate_id = line + self.poi_cate_vocab[cate] = int(cate_id) + + + + mean = (0.38247773301793103, 0.41271937512626544, 0.3403674447902101) + std = (0.17229526604561052, 0.14788625733896113, 0.16325427643628246) + self.img_trans = paddle.vision.transforms.Normalize(mean, std) + + + + def tokenize_poi_data(self, index): + poi_list = self.poi_zorder_list[index] + region_coord = self.region_coord_list[index] + + if poi_list is None: + token_id_seq = [self.cls_token_id] + [self.pad_token_id] * (self.max_len_token - 1) + attn_mask_seq = [1] + [0] * (self.max_len_token - 1) + word_level_pos_id_seq = [0]+ [i % self.max_len_token for i in range(self.max_len_token - 1)] + poi_level_pos_id_seq = [0] + [1 + int(i / self.max_len_token) for i in range(self.max_len_token - 1)] + grid_level_pos_id_seq = [0] + [self.num_grid + 1] * (self.max_len_token - 1) + offset = [1] + poi_cate_id_seq = [self.poi_cate_vocab['PAD']] * self.max_len_token + + + else: + cur_len, num_poi = 1, 1 + token_id_seq, attn_mask_seq = [self.cls_token_id], [1] + word_level_pos_id_seq, poi_level_pos_id_seq, grid_level_pos_id_seq = [0], [0], [0] + offset = [cur_len] + poi_cate_id_seq = [self.poi_cate_vocab['PAD']] + + + for poi in poi_list: + poi_name, _, poi_cate_id, poi_x, poi_y = poi + + poi_name = poi_name.lower() + tokenize_output = self.tokenizer(poi_name, add_special_tokens=False) + + token_id = tokenize_output['input_ids'] + attn_mask = tokenize_output['attention_mask'] + + token_id.append(self.sep_token_id) # add sep token behind each poi + attn_mask.append(1) + + word_level_pos_id = list(range(len(token_id))) + + poi_level_pos_id = [num_poi] * len(token_id) + + grid_x = int(self.num_grid_x * (poi_x - region_coord[0]) / (region_coord[2] - region_coord[0])) + grid_y = int(self.num_grid_y * (region_coord[1] - poi_y) / (region_coord[1] - region_coord[3])) + grid_x, grid_y = min(grid_x, self.num_grid_x - 1), min(grid_y, self.num_grid_y - 1) + grid_id = self.num_grid_x * grid_y + grid_x + 1 + grid_level_pos_id = [grid_id] * len(token_id) + + poi_cate_id = [poi_cate_id] * len(token_id) + + cur_len += len(token_id) + if cur_len <= self.max_len_token: + offset.append(cur_len) + token_id_seq.extend(token_id) + attn_mask_seq.extend(attn_mask) + word_level_pos_id_seq.extend(word_level_pos_id) + poi_level_pos_id_seq.extend(poi_level_pos_id) + grid_level_pos_id_seq.extend(grid_level_pos_id) + poi_cate_id_seq.extend(poi_cate_id) + num_poi += 1 + + else: + break + + ## padding + padding_len = self.max_len_token - offset[-1] + if padding_len > 0: + token_id_seq.extend([self.pad_token_id] * padding_len) + attn_mask_seq.extend([0] * padding_len) + word_level_pos_id_seq.extend([i % self.max_len_token for i in range(padding_len)]) + poi_level_pos_id_seq.extend([num_poi + int(i / self.max_len_poi) for i in range(padding_len)]) + grid_level_pos_id_seq.extend([self.num_grid + 1] * padding_len) + poi_cate_id_seq.extend([self.poi_cate_vocab['PAD']] * padding_len) + + return token_id_seq, attn_mask_seq, word_level_pos_id_seq, poi_level_pos_id_seq, grid_level_pos_id_seq, poi_cate_id_seq + + + + def __getitem__(self, index): + id = self.id_of_region[index] + + poi_name_token_ids, attn_mask_poi, word_level_pos_ids, poi_level_pos_ids, grid_level_pos_ids, poi_cate_ids = self.tokenize_poi_data(index) + + img = pil_loader(self.img_path + '{id}.png'.format(id=id)) + img = np.array(img).astype('float32') / 255.0 + img = img.transpose(2, 0, 1) + img = self.img_trans(img) + + poi_name_token_ids = np.array(poi_name_token_ids) + attn_mask_poi = np.array(attn_mask_poi) + word_level_pos_ids = np.array(word_level_pos_ids) + poi_level_pos_ids = np.array(poi_level_pos_ids) + grid_level_pos_ids = np.array(grid_level_pos_ids) + poi_cate_ids = np.array(poi_cate_ids) + + return poi_name_token_ids, attn_mask_poi, word_level_pos_ids, poi_level_pos_ids, grid_level_pos_ids, poi_cate_ids, img + + + def __len__(self): + return len(self.id_of_region) + + + \ No newline at end of file diff --git a/research/ReFound/code/feature_based_cap.py b/research/ReFound/code/feature_based_cap.py new file mode 100644 index 0000000..80b49c7 --- /dev/null +++ b/research/ReFound/code/feature_based_cap.py @@ -0,0 +1,242 @@ +# -*- coding: utf-8 -*- + +import os +import json +from symbol import parameters +import PIL.Image as pil +import random +import time +import datetime +import numpy as np +import paddle +import paddle.nn as nn +from paddle.io import Dataset, DataLoader +import paddle.optimizer as optim +from collections import OrderedDict +from sklearn.metrics import roc_curve, auc, precision_recall_curve +from sklearn.metrics import mean_squared_error, mean_absolute_error, r2_score +import argparse +from utils import * +from models import FeatureBasedCommercialActivenessPrediction +from configuration import config + + +class FeatureBased_Dataset_CAP(Dataset): + def __init__(self, config=None, dataset_type=None): + super().__init__() + self.config = config + file_path = os.path.abspath(__file__) + self.root_path = os.path.dirname(file_path) + '/../' + self.dataset_type = dataset_type + self.id_of_region_path = self.root_path + 'data/cap_{c}_id_of_region_{dt}'.format(c=config['city'], dt=dataset_type) + self.label_path = self.root_path + 'data/cap_{c}_label_{dt}'.format(c=config['city'], dt=dataset_type) + self.region_emb_path = self.root_path + 'region_emb/{c}.embed'.format(c=config['city']) + + self.rehash_id_of_region = [] + with open(self.id_of_region_path, 'r') as f: + for line in f: + line = eval(line.strip('\n')) + rehash_id = int(line[0]) + self.rehash_id_of_region.append(rehash_id) + + label_list = [] + with open(self.label_path, 'r') as f: + for line in f: + line = eval(line.strip('\n')) + label_list.append(line) + + self.label_list = [float(label) for label in label_list] + + region_emb_total = paddle.load(self.region_emb_path) + self.region_emb = region_emb_total[self.rehash_id_of_region] + + + def __getitem__(self, index): + label = self.label_list[index] + region_emb = self.region_emb[index] + return label, region_emb + + + def __len__(self): + return len(self.rehash_id_of_region) + + + + +class Trainer(object): + def __init__(self, config): + super(Trainer, self).__init__() + self.config = config + file_path = os.path.abspath(__file__) + self.log_path = os.path.dirname(file_path) + '/../log_feature/CAP/{c}/'.format(c=config['city']) + + self.param_info = '{agg}_drop{fdrop}_bs{fbs}x{ac}_lr{flr}_dc{fdc}_ep{ep}.{warm}_seed{sd}'.format( + agg=config['agg'], fdrop=config['fdrop'], fbs=config['fbatch_size'], flr=config['flr'], fdc=config['fdecay'], + ac=config['accum_iter'], ep=config['epoch_num'], warm=config['warmup_epochs'], sd=config['seed'] + ) + + def get_dataloader(self): + train_dataset = FeatureBased_Dataset_CAP(config=self.config, dataset_type='train') + val_dataset = FeatureBased_Dataset_CAP(config=self.config, dataset_type='val') + test_dataset = FeatureBased_Dataset_CAP(config=self.config, dataset_type='test') + train_loader = DataLoader(dataset=train_dataset, batch_size=self.config['fbatch_size'], shuffle=True) + val_loader = DataLoader(dataset=val_dataset, batch_size=self.config['fbatch_size'], shuffle=False) + test_loader = DataLoader(dataset=test_dataset, batch_size=self.config['fbatch_size'], shuffle=False) + print(len(train_dataset), len(train_loader), len(val_dataset), len(val_loader), len(test_dataset), len(test_loader)) + return train_loader, val_loader, test_loader + + + def load_model(self): + model = FeatureBasedCommercialActivenessPrediction(config=self.config) + return model + + + def train(self): + train_loader, val_loader, test_loader = self.get_dataloader() + model = self.load_model() + + optimizer = optim.AdamW(parameters=model.parameters(), learning_rate=self.config['flr'], weight_decay=self.config['fdecay']) + criterion = nn.MSELoss() + best_rmse_val, best_epoch = 999999, 0 + + self.start_time = time.time() + for epoch in range(self.config['epoch_num']): + self.train_one_epoch( + model=model, + train_loader=train_loader, + optimizer=optimizer, + epoch=epoch, + criterion=criterion, + ) + + rmse_val, mae_val, R2_val = self.evaluate( + model=model, + eval_loader=val_loader, + ) + + rmse_test, mae_test, R2_test = self.evaluate( + model=model, + eval_loader=test_loader, + ) + + if rmse_val < best_rmse_val: + best_epoch = epoch + best_rmse_val, best_mae_val, best_R2_val = rmse_val, mae_val, R2_val + best_rmse_test, best_mae_test, best_R2_test = rmse_test, mae_test, R2_test + star = '*** ' + else: star = '' + + log_new = star + "Epoch: %3d | Val RMSE: %.4f | Val MAE: %.4f | Val R2: %.4f | Test RMSE: %.4f | Test MAE: %.4f | Test R2: %.4f \n" \ + % (epoch, rmse_val, mae_val, R2_val, rmse_test, mae_test, R2_test) + self.log = self.log + log_new + '\n' + print(log_new) + + log_new = "Best Epoch: %3d | Best Val RMSE: %.4f | Best Val MAE: %.4f | Best Val R2: %.4f | Best Test RMSE: %.4f | Best Test MAE: %.4f | best test R2: %.4f " \ + % (best_epoch, best_rmse_val, best_mae_val, best_R2_val, best_rmse_test, best_mae_test, best_R2_test) + self.log = self.log + log_new + '\n' + print(log_new) + + + def train_one_epoch(self, model, train_loader, optimizer, epoch, criterion): + model.train() + optimizer.clear_grad() + + for step, data in enumerate(train_loader): + label, region_emb = data + + label = label.cast('float32') + + if step % self.config['accum_iter'] == 0: + adjust_learning_rate( + optimizer=optimizer, + epoch=step / len(train_loader) + epoch, + warmup_epochs=self.config['warmup_epochs'], + epoch_num=self.config['epoch_num'], + peak_lr=self.config['flr'], + min_lr=self.config['min_lr']) + + pred = model(region_emb=region_emb) + pred = pred.squeeze(-1) + loss = criterion(pred, label) + + loss_value = loss.item() + + loss = loss / self.config['accum_iter'] + loss.backward() + + if (step + 1) % self.config['accum_iter'] == 0: + optimizer.step() + optimizer.clear_grad() + + paddle.device.cuda.synchronize() + total_time = time.time() - self.start_time + total_time = str(datetime.timedelta(seconds=int(total_time))) + + if step % config['logging_step'] == 0: + log_new = "Epoch: %3d | Step: %4d | Train Loss: %7.4f | Time: %s" % (epoch, step, loss_value, total_time) + self.log = self.log + log_new + '\n' + print(log_new) + + + @paddle.no_grad() + def evaluate(self, model, eval_loader): + eval_pred_list, eval_label_list = [], [] + model.eval() + for _, data in enumerate(eval_loader): + label, region_emb = data + + eval_label_list += label.tolist() + + pred = model(region_emb=region_emb) + pred = pred.squeeze(-1) + eval_pred_list += pred.cpu().tolist() + + rmse_eval = np.sqrt(mean_squared_error(eval_label_list, eval_pred_list)) + mae_eval = mean_absolute_error(eval_label_list, eval_pred_list) + R2_eval = r2_score(eval_label_list, eval_pred_list) + return rmse_eval, mae_eval, R2_eval + + + def Train(self): + seed_setup(self.config['seed']) + self.log = str(self.config) + '\n------------------- start training ----------------------\n' + self.train() + self.write_log() + + + def write_log(self): + log_output_path = self.log_path + self.param_info + if os.path.exists(log_output_path): + os.system('rm ' + log_output_path) + with open(log_output_path, 'w') as f: + f.write(self.log) + + +if __name__ == '__main__': + pass + parser = argparse.ArgumentParser() + parser.add_argument('--city', type=str, default=None) + parser.add_argument('--checkpoint', type=int, default=None) + parser.add_argument('--agg', type=str, default=None) + parser.add_argument('--fdrop', type=float, default=None) + parser.add_argument('--fbatch_size', type=int, default=None) + parser.add_argument('--epoch_num', type=int, default=None) + parser.add_argument('--warmup_epochs', type=int, default=None) + parser.add_argument('--flr', type=float, default=None) + parser.add_argument('--fdecay', type=float, default=None) + parser.add_argument('--min_lr', type=float, default=None) + parser.add_argument('--accum_iter', type=int, default=None) + parser.add_argument('--seed', type=int, default=42) + parser.add_argument('--logging_step', type=int, default=10) + args = parser.parse_args() + + for k, v in vars(args).items(): + config[k] = v + + for k, v in config.items(): + print('{}: '.format(k), v, type(v)) + + os.environ['TOKENIZERS_PARALLELISM'] = 'false' + + Trainer_ = Trainer(config=config) + Trainer_.Train() \ No newline at end of file diff --git a/research/ReFound/code/feature_based_pop.py b/research/ReFound/code/feature_based_pop.py new file mode 100644 index 0000000..e7ce3f0 --- /dev/null +++ b/research/ReFound/code/feature_based_pop.py @@ -0,0 +1,242 @@ +# -*- coding: utf-8 -*- + +import os +import json +from symbol import parameters +import PIL.Image as pil +import random +import time +import datetime +import numpy as np +import paddle +import paddle.nn as nn +from paddle.io import Dataset, DataLoader +import paddle.optimizer as optim +from collections import OrderedDict +from sklearn.metrics import roc_curve, auc, precision_recall_curve +from sklearn.metrics import mean_squared_error, mean_absolute_error, r2_score +import argparse +from utils import * +from models import FeatureBasedPopulationPrediction +from configuration import config + + +class FeatureBased_Dataset_POP(Dataset): + def __init__(self, config=None, dataset_type=None): + super().__init__() + self.config = config + file_path = os.path.abspath(__file__) + self.root_path = os.path.dirname(file_path) + '/../' + self.dataset_type = dataset_type + self.id_of_region_path = self.root_path + 'data/pop_{c}_id_of_region_{dt}'.format(c=config['city'], dt=dataset_type) + self.label_path = self.root_path + 'data/pop_{c}_label_{dt}'.format(c=config['city'], dt=dataset_type) + self.region_emb_path = self.root_path + 'region_emb/{c}.embed'.format(c=config['city']) + + self.rehash_id_of_region = [] + with open(self.id_of_region_path, 'r') as f: + for line in f: + line = eval(line.strip('\n')) + rehash_id = int(line[0]) + self.rehash_id_of_region.append(rehash_id) + + label_list = [] + with open(self.label_path, 'r') as f: + for line in f: + line = eval(line.strip('\n')) + label_list.append(line) + + self.label_list = [float(label) for label in label_list] + + region_emb_total = paddle.load(self.region_emb_path) + self.region_emb = region_emb_total[self.rehash_id_of_region] + + + def __getitem__(self, index): + label = self.label_list[index] + region_emb = self.region_emb[index] + return label, region_emb + + + def __len__(self): + return len(self.rehash_id_of_region) + + + + +class Trainer(object): + def __init__(self, config): + super(Trainer, self).__init__() + self.config = config + file_path = os.path.abspath(__file__) + self.log_path = os.path.dirname(file_path) + '/../log_feature/POP/{c}/'.format(c=config['city']) + + self.param_info = '{agg}_drop{fdrop}_bs{fbs}x{ac}_lr{flr}_dc{fdc}_ep{ep}.{warm}_seed{sd}'.format( + agg=config['agg'], fdrop=config['fdrop'], fbs=config['fbatch_size'], flr=config['flr'], fdc=config['fdecay'], + ac=config['accum_iter'], ep=config['epoch_num'], warm=config['warmup_epochs'], sd=config['seed'] + ) + + def get_dataloader(self): + train_dataset = FeatureBased_Dataset_POP(config=self.config, dataset_type='train') + val_dataset = FeatureBased_Dataset_POP(config=self.config, dataset_type='val') + test_dataset = FeatureBased_Dataset_POP(config=self.config, dataset_type='test') + train_loader = DataLoader(dataset=train_dataset, batch_size=self.config['fbatch_size'], shuffle=True) + val_loader = DataLoader(dataset=val_dataset, batch_size=self.config['fbatch_size'], shuffle=False) + test_loader = DataLoader(dataset=test_dataset, batch_size=self.config['fbatch_size'], shuffle=False) + print(len(train_dataset), len(train_loader), len(val_dataset), len(val_loader), len(test_dataset), len(test_loader)) + return train_loader, val_loader, test_loader + + + def load_model(self): + model = FeatureBasedPopulationPrediction(config=self.config) + return model + + + def train(self): + train_loader, val_loader, test_loader = self.get_dataloader() + model = self.load_model() + + optimizer = optim.AdamW(parameters=model.parameters(), learning_rate=self.config['flr'], weight_decay=self.config['fdecay']) + criterion = nn.MSELoss() + best_rmse_val, best_epoch = 999999, 0 + + self.start_time = time.time() + for epoch in range(self.config['epoch_num']): + self.train_one_epoch( + model=model, + train_loader=train_loader, + optimizer=optimizer, + epoch=epoch, + criterion=criterion, + ) + + rmse_val, mae_val, R2_val = self.evaluate( + model=model, + eval_loader=val_loader, + ) + + rmse_test, mae_test, R2_test = self.evaluate( + model=model, + eval_loader=test_loader, + ) + + if rmse_val < best_rmse_val: + best_epoch = epoch + best_rmse_val, best_mae_val, best_R2_val = rmse_val, mae_val, R2_val + best_rmse_test, best_mae_test, best_R2_test = rmse_test, mae_test, R2_test + star = '*** ' + else: star = '' + + log_new = star + "Epoch: %3d | Val RMSE: %.4f | Val MAE: %.4f | Val R2: %.4f | Test RMSE: %.4f | Test MAE: %.4f | Test R2: %.4f \n" \ + % (epoch, rmse_val, mae_val, R2_val, rmse_test, mae_test, R2_test) + self.log = self.log + log_new + '\n' + print(log_new) + + log_new = "Best Epoch: %3d | Best Val RMSE: %.4f | Best Val MAE: %.4f | Best Val R2: %.4f | Best Test RMSE: %.4f | Best Test MAE: %.4f | best test R2: %.4f " \ + % (best_epoch, best_rmse_val, best_mae_val, best_R2_val, best_rmse_test, best_mae_test, best_R2_test) + self.log = self.log + log_new + '\n' + print(log_new) + + + def train_one_epoch(self, model, train_loader, optimizer, epoch, criterion): + model.train() + optimizer.clear_grad() + + for step, data in enumerate(train_loader): + label, region_emb = data + + label = label.cast('float32') + + if step % self.config['accum_iter'] == 0: + adjust_learning_rate( + optimizer=optimizer, + epoch=step / len(train_loader) + epoch, + warmup_epochs=self.config['warmup_epochs'], + epoch_num=self.config['epoch_num'], + peak_lr=self.config['flr'], + min_lr=self.config['min_lr']) + + pred = model(region_emb=region_emb) + pred = pred.squeeze(-1) + loss = criterion(pred, label) + + loss_value = loss.item() + + loss = loss / self.config['accum_iter'] + loss.backward() + + if (step + 1) % self.config['accum_iter'] == 0: + optimizer.step() + optimizer.clear_grad() + + paddle.device.cuda.synchronize() + total_time = time.time() - self.start_time + total_time = str(datetime.timedelta(seconds=int(total_time))) + + if step % config['logging_step'] == 0: + log_new = "Epoch: %3d | Step: %4d | Train Loss: %7.4f | Time: %s" % (epoch, step, loss_value, total_time) + self.log = self.log + log_new + '\n' + print(log_new) + + + @paddle.no_grad() + def evaluate(self, model, eval_loader): + eval_pred_list, eval_label_list = [], [] + model.eval() + for _, data in enumerate(eval_loader): + label, region_emb = data + + eval_label_list += label.tolist() + + pred = model(region_emb=region_emb) + pred = pred.squeeze(-1) + eval_pred_list += pred.cpu().tolist() + + rmse_eval = np.sqrt(mean_squared_error(eval_label_list, eval_pred_list)) + mae_eval = mean_absolute_error(eval_label_list, eval_pred_list) + R2_eval = r2_score(eval_label_list, eval_pred_list) + return rmse_eval, mae_eval, R2_eval + + + def Train(self): + seed_setup(self.config['seed']) + self.log = str(self.config) + '\n------------------- start training ----------------------\n' + self.train() + self.write_log() + + + def write_log(self): + log_output_path = self.log_path + self.param_info + if os.path.exists(log_output_path): + os.system('rm ' + log_output_path) + with open(log_output_path, 'w') as f: + f.write(self.log) + + +if __name__ == '__main__': + pass + parser = argparse.ArgumentParser() + parser.add_argument('--city', type=str, default=None) + parser.add_argument('--checkpoint', type=int, default=None) + parser.add_argument('--agg', type=str, default=None) + parser.add_argument('--fdrop', type=float, default=None) + parser.add_argument('--fbatch_size', type=int, default=None) + parser.add_argument('--epoch_num', type=int, default=None) + parser.add_argument('--warmup_epochs', type=int, default=None) + parser.add_argument('--flr', type=float, default=None) + parser.add_argument('--fdecay', type=float, default=None) + parser.add_argument('--min_lr', type=float, default=None) + parser.add_argument('--accum_iter', type=int, default=None) + parser.add_argument('--seed', type=int, default=42) + parser.add_argument('--logging_step', type=int, default=10) + args = parser.parse_args() + + for k, v in vars(args).items(): + config[k] = v + + for k, v in config.items(): + print('{}: '.format(k), v, type(v)) + + os.environ['TOKENIZERS_PARALLELISM'] = 'false' + + Trainer_ = Trainer(config=config) + Trainer_.Train() \ No newline at end of file diff --git a/research/ReFound/code/feature_based_uvd.py b/research/ReFound/code/feature_based_uvd.py new file mode 100644 index 0000000..d8f8bc2 --- /dev/null +++ b/research/ReFound/code/feature_based_uvd.py @@ -0,0 +1,275 @@ +# -*- coding: utf-8 -*- + +import os +import json +import PIL.Image as pil +import random +import time +import datetime +import numpy as np +import paddle +import paddle.nn as nn +from paddle.io import Dataset, DataLoader +import paddle.optimizer as optim +from sklearn.metrics import roc_curve, auc, precision_recall_curve +from sklearn.metrics import mean_squared_error, mean_absolute_error, r2_score +from collections import OrderedDict +import argparse +from utils import * +from models import FeatureBasedUrbanVillageDetection +from configuration import config + + + +class FeatureBased_Dataset_UV(Dataset): + def __init__(self, config=None, dataset_type=None): + super().__init__() + self.config = config + file_path = os.path.abspath(__file__) + self.root_path = os.path.dirname(file_path) + '/../' + self.dataset_type = dataset_type + self.id_of_region_path = self.root_path + 'data/uvd_{c}_id_of_region_{dt}'.format(c=config['city'], dt=dataset_type) + self.label_path = self.root_path + 'data/uvd_{c}_label_{dt}'.format(c=config['city'], dt=dataset_type) + self.region_emb_path = self.root_path + 'region_emb/{c}.embed'.format(c=config['city']) + + self.rehash_id_of_region = [] + with open(self.id_of_region_path, 'r') as f: + for line in f: + line = eval(line.strip('\n')) + rehash_id = int(line[0]) + self.rehash_id_of_region.append(rehash_id) + + label_list = [] + with open(self.label_path, 'r') as f: + for line in f: + line = eval(line.strip('\n')) + label_list.append(line) + + self.label_list = [float(label) for label in label_list] + + region_emb_total = paddle.load(self.region_emb_path) + self.region_emb = region_emb_total[self.rehash_id_of_region] + + + def __getitem__(self, index): + label = self.label_list[index] + region_emb = self.region_emb[index] + return label, region_emb + + + def __len__(self): + return len(self.rehash_id_of_region) + + + + +class Trainer(object): + def __init__(self, config): + super(Trainer, self).__init__() + self.config = config + file_path = os.path.abspath(__file__) + self.log_path = os.path.dirname(file_path) + '/../log_feature/UVD/{c}/'.format(c=config['city']) + self.prob_path = os.path.dirname(file_path) + '/../prob_label_feature/UVD/{c}/'.format(c=config['city']) + + self.param_info = '{agg}_drop{fdrop}_bs{fbs}x{ac}_lr{flr}_dc{fdc}_ep{ep}.{warm}_seed{sd}'.format( + agg=config['agg'], fdrop=config['fdrop'], fbs=config['fbatch_size'], flr=config['flr'], fdc=config['fdecay'], + ac=config['accum_iter'], ep=config['epoch_num'], warm=config['warmup_epochs'], sd=config['seed'] + ) + + def get_dataloader(self): + train_dataset = FeatureBased_Dataset_UV(config=self.config, dataset_type='train') + val_dataset = FeatureBased_Dataset_UV(config=self.config, dataset_type='val') + test_dataset = FeatureBased_Dataset_UV(config=self.config, dataset_type='test') + train_loader = DataLoader(dataset=train_dataset, batch_size=self.config['fbatch_size'], shuffle=True) + train_loader_for_eval = DataLoader(dataset=train_dataset, batch_size=self.config['fbatch_size'], shuffle=False) + val_loader = DataLoader(dataset=val_dataset, batch_size=self.config['fbatch_size'], shuffle=False) + test_loader = DataLoader(dataset=test_dataset, batch_size=self.config['fbatch_size'], shuffle=False) + print(len(train_dataset), len(train_loader), len(val_dataset), len(val_loader), len(test_dataset), len(test_loader)) + return train_dataset, val_dataset, test_dataset, train_loader, train_loader_for_eval, val_loader, test_loader + + + def load_model(self): + model = FeatureBasedUrbanVillageDetection(config=self.config) + return model + + + def train(self, save_epoch=-1): + train_dataset, val_dataset, test_dataset, train_loader, train_loader_for_eval, val_loader, test_loader = self.get_dataloader() + model = self.load_model() + + optimizer = optim.AdamW(parameters=model.parameters(), learning_rate=self.config['flr'], weight_decay=self.config['fdecay']) + criterion = nn.BCELoss() + best_AUC_val, best_epoch = 0, 0 + + prob_label_dict = {'prob': {}, 'label': {}} + prob_label_dict['label']['train'] = train_dataset.label_list + prob_label_dict['label']['val'] = val_dataset.label_list + prob_label_dict['label']['test'] = test_dataset.label_list + + self.start_time = time.time() + for epoch in range(self.config['epoch_num']): + self.train_one_epoch( + model=model, + train_loader=train_loader, + optimizer=optimizer, + epoch=epoch, + criterion=criterion, + ) + + AUC_train, train_prob_list = self.evaluate( + model=model, + eval_loader=train_loader_for_eval, + ) + + AUC_val, val_prob_list = self.evaluate( + model=model, + eval_loader=val_loader, + ) + + AUC_test, test_prob_list = self.evaluate( + model=model, + eval_loader=test_loader, + ) + + if AUC_val > best_AUC_val: + best_AUC_val = AUC_val + best_epoch = epoch + best_AUC_test = AUC_test + star = '*** ' + + prob_label_dict['prob']['train'] = train_prob_list + prob_label_dict['prob']['val'] = val_prob_list + prob_label_dict['prob']['test'] = test_prob_list + + else: star = '' + + log_new = star + "Epoch: %3d | Val AUC: %.4f | Test AUC: %.4f \n" % (epoch, AUC_val, AUC_test) + self.log = self.log + log_new + '\n' + print(log_new) + + if epoch == save_epoch: + self.save_model(model) + break + + log_new = "Best Epoch: %3d | Best Val AUC: %.4f | Best Test AUC: %.4f" % (best_epoch, best_AUC_val, best_AUC_test) + self.log = self.log + log_new + '\n' + print(log_new) + + self.save_prob_label(prob_label_dict=prob_label_dict) + + return best_epoch + + + def train_one_epoch(self, model, train_loader, optimizer, epoch, criterion): + model.train() + optimizer.clear_grad() + + for step, data in enumerate(train_loader): + label, region_emb = data + + label = label.cast('float32') + + if step % self.config['accum_iter'] == 0: + adjust_learning_rate( + optimizer=optimizer, + epoch=step / len(train_loader) + epoch, + warmup_epochs=self.config['warmup_epochs'], + epoch_num=self.config['epoch_num'], + peak_lr=self.config['flr'], + min_lr=self.config['min_lr']) + + prob = model(region_emb=region_emb) + prob = prob.squeeze(-1) + loss = criterion(prob, label) + + loss_value = loss.item() + + loss = loss / self.config['accum_iter'] + loss.backward() + + if (step + 1) % self.config['accum_iter'] == 0: + optimizer.step() + optimizer.clear_grad() + + paddle.device.cuda.synchronize() + total_time = time.time() - self.start_time + total_time = str(datetime.timedelta(seconds=int(total_time))) + + if step % config['logging_step'] == 0: + log_new = "Epoch: %3d | Step: %4d | Train Loss: %7.4f | Time: %s" % (epoch, step, loss_value, total_time) + self.log = self.log + log_new + '\n' + print(log_new) + + + @paddle.no_grad() + def evaluate(self, model, eval_loader): + eval_prob_list, eval_label_list = [], [] + model.eval() + for _, data in enumerate(eval_loader): + label, region_emb = data + + eval_label_list += label.tolist() + + prob = model(region_emb=region_emb) + prob = prob.squeeze(-1) + eval_prob_list += prob.cpu().tolist() + + fpr, tpr, _ = roc_curve(eval_label_list, eval_prob_list, pos_label=1) + AUC_eval = auc(fpr, tpr) + return AUC_eval, eval_prob_list + + + def Train(self): + seed_setup(self.config['seed']) + self.log = str(self.config) + '\n------------------- start training ----------------------\n' + best_epoch = self.train() + self.write_log() + + + def write_log(self): + log_output_path = self.log_path + self.param_info + if os.path.exists(log_output_path): + os.system('rm ' + log_output_path) + with open(log_output_path, 'w') as f: + f.write(self.log) + + + def save_model(self, model): + save_dir = self.model_path + paddle.save(model.state_dict(), save_dir) + + + def save_prob_label(self, prob_label_dict): + prob_label_path = self.prob_path + self.param_info + with open(prob_label_path, 'w') as f: + f.write(str(prob_label_dict)) + + +if __name__ == '__main__': + pass + parser = argparse.ArgumentParser() + parser.add_argument('--city', type=str, default=None) + parser.add_argument('--checkpoint', type=int, default=None) + parser.add_argument('--agg', type=str, default=None) + parser.add_argument('--fdrop', type=float, default=None) + parser.add_argument('--fbatch_size', type=int, default=None) + parser.add_argument('--epoch_num', type=int, default=None) + parser.add_argument('--warmup_epochs', type=int, default=None) + parser.add_argument('--flr', type=float, default=None) + parser.add_argument('--fdecay', type=float, default=None) + parser.add_argument('--min_lr', type=float, default=None) + parser.add_argument('--accum_iter', type=int, default=None) + parser.add_argument('--seed', type=int, default=42) + parser.add_argument('--logging_step', type=int, default=10) + args = parser.parse_args() + + for k, v in vars(args).items(): + config[k] = v + + for k, v in config.items(): + print('{}: '.format(k), v, type(v)) + + os.environ['TOKENIZERS_PARALLELISM'] = 'false' + + Trainer_ = Trainer(config=config) + Trainer_.Train() \ No newline at end of file diff --git a/research/ReFound/code/feature_extraction.sh b/research/ReFound/code/feature_extraction.sh new file mode 100644 index 0000000..ae43204 --- /dev/null +++ b/research/ReFound/code/feature_extraction.sh @@ -0,0 +1,14 @@ +#!/bin/bash + +shparam1=${1} + +function region_feature_extraction() { + CUDA_VISIBLE_DEVICES=0 \ + python -u feature_extractor.py \ + --city ${1} \ + --checkpoint '299' \ + --extract_batch_size '128' \ + --seed '42' +} + +region_feature_extraction ${shparam1} \ No newline at end of file diff --git a/research/ReFound/code/feature_extractor.py b/research/ReFound/code/feature_extractor.py new file mode 100644 index 0000000..43f16b0 --- /dev/null +++ b/research/ReFound/code/feature_extractor.py @@ -0,0 +1,130 @@ +# -*- coding: utf-8 -*- + +import os +import json +import PIL.Image as pil +import random +import time +import datetime +import numpy as np +import paddle +import paddle.nn as nn +from paddle.io import DataLoader +import paddle.optimizer as optim +from collections import OrderedDict +import argparse +from utils import * +from dataset_feature import * +from models import FeatureExtractor +from configuration import config + + +class Trainer(object): + def __init__(self, config): + super(Trainer, self).__init__() + self.config = config + file_path = os.path.abspath(__file__) + self.bert_chinese_path = os.path.dirname(file_path) + '/../bert-base-chinese/' + self.region_emb_save_path = os.path.dirname(file_path) + '/../region_emb/{c}.embed'.format(c=config['city']) + self.pretrain_state_path = os.path.dirname(file_path) + '/../checkpoint/' + + + def get_dataloader(self): + dataset = Dataset_RegionFeature(config=self.config) + dataloader = DataLoader(dataset=dataset, batch_size=self.config['extract_batch_size'], shuffle=False, num_workers=4) + print(len(dataset), len(dataloader)) + return dataloader + + + def load_model(self): + model = FeatureExtractor(config=self.config) + checkpoint_path = self.pretrain_state_path + 'checkpoint-{}.pdparams'.format(self.config['checkpoint']) + print("checkpoint_path:\n" + checkpoint_path) + pretrain_state = paddle.load(checkpoint_path) + model.set_state_dict(pretrain_state) + return model + + + def train(self): + dataloader = self.get_dataloader() + model = self.load_model() + + self.feature_extraction( + model=model, + dataloader=dataloader, + ) + + + @paddle.no_grad() + def feature_extraction(self, model, dataloader): + region_emb_list = [] + model.eval() + + for step, data in enumerate(dataloader): + ( + poi_name_token_ids, + attn_mask_poi, + word_level_pos_ids, + poi_level_pos_ids, + grid_level_pos_ids, + poi_cate_ids, + img + ) = data + + poi_data = { + 'poi_name_token_ids': poi_name_token_ids, + 'attn_mask_poi': attn_mask_poi, + 'word_level_pos_ids': word_level_pos_ids, + 'poi_level_pos_ids': poi_level_pos_ids, + 'grid_level_pos_ids': grid_level_pos_ids, + 'poi_cate_ids': poi_cate_ids, + } + + img_data = { + 'img': img, + } + + region_emb = model(poi_data=poi_data, img_data=img_data) + region_emb_list.append(region_emb.cpu()) + + print("extract {} steps".format(step)) + + + region_emb_list = paddle.concat(region_emb_list, axis=0) + print(region_emb_list.shape) + self.save_region_emb(region_emb_list=region_emb_list) + + + def Train(self): + seed_setup(self.config['seed']) + self.log = str(self.config) + '\n------------------- start training ----------------------\n' + self.train() + + + def save_region_emb(self, region_emb_list): + region_emb_save_path = self.region_emb_save_path + if os.path.exists(region_emb_save_path): + os.system('rm ' + region_emb_save_path) + + paddle.save(region_emb_list, region_emb_save_path) + + +if __name__ == '__main__': + pass + parser = argparse.ArgumentParser() + parser.add_argument('--city', type=str, default=None) + parser.add_argument('--checkpoint', type=int, default=None) + parser.add_argument('--extract_batch_size', type=int, default=None) + parser.add_argument('--seed', type=int, default=None) + args = parser.parse_args() + + for k, v in vars(args).items(): + config[k] = v + + for k, v in config.items(): + print('{}: '.format(k), v, type(v)) + + os.environ['TOKENIZERS_PARALLELISM'] = 'false' + + Trainer_ = Trainer(config=config) + Trainer_.Train() \ No newline at end of file diff --git a/research/ReFound/code/finetune_cap.py b/research/ReFound/code/finetune_cap.py new file mode 100644 index 0000000..faf276a --- /dev/null +++ b/research/ReFound/code/finetune_cap.py @@ -0,0 +1,256 @@ +# -*- coding: utf-8 -*- + +import os +import json +import PIL.Image as pil +import random +import time +import datetime +import numpy as np +import paddle +import paddle.nn as nn +from paddle.io import DataLoader +import paddle.optimizer as optim +from sklearn.metrics import roc_curve, auc, precision_recall_curve +from sklearn.metrics import mean_squared_error, mean_absolute_error, r2_score +from collections import OrderedDict +import argparse +from utils import * +from ft_dataset_cap import * +from models import FinetuneCommercialActivenessPrediction +from configuration import config + +class Trainer(object): + def __init__(self, config): + super(Trainer, self).__init__() + self.config = config + file_path = os.path.abspath(__file__) + self.bert_chinese_path = os.path.dirname(file_path) + '/../bert-base-chinese/' + self.log_path = os.path.dirname(file_path) + '/../log/CAP/{c}/'.format(c=config['city']) + self.pretrain_state_path = os.path.dirname(file_path) + '/../checkpoint/' + + self.param_info = '{agg}_drop{fdrop}_bs{fbs}x{ac}_lr{flr}_dc{fdc}_ep{ep}.{warm}_seed{sd}'.format( + agg=config['agg'], fdrop=config['fdrop'], fbs=config['fbatch_size'], flr=config['flr'], fdc=config['fdecay'], + ac=config['accum_iter'], ep=config['epoch_num'], warm=config['warmup_epochs'], sd=config['seed'] + ) + + def get_dataloader(self): + train_dataset = FT_Dataset_CAP(config=self.config, dataset_type='train') + val_dataset = FT_Dataset_CAP(config=self.config, dataset_type='val') + test_dataset = FT_Dataset_CAP(config=self.config, dataset_type='test') + train_loader = DataLoader(dataset=train_dataset, batch_size=self.config['fbatch_size'], shuffle=True, num_workers=1) + val_loader = DataLoader(dataset=val_dataset, batch_size=self.config['fbatch_size'], shuffle=False, num_workers=1) + test_loader = DataLoader(dataset=test_dataset, batch_size=self.config['fbatch_size'], shuffle=False, num_workers=1) + print(len(train_dataset), len(train_loader), len(val_dataset), len(val_loader), len(test_dataset), len(test_loader)) + return train_loader, val_loader, test_loader + + + def load_model(self): + model = FinetuneCommercialActivenessPrediction(config=self.config) + checkpoint_path = self.pretrain_state_path + 'checkpoint-{}.pdparams'.format(self.config['checkpoint']) + print("checkpoint_path:\n" + checkpoint_path) + pretrain_state = paddle.load(checkpoint_path) + model.set_state_dict(pretrain_state) + return model + + + def train(self): + train_loader, val_loader, test_loader = self.get_dataloader() + model = self.load_model() + + param_groups = param_groups_lrd(model=model, weight_decay=self.config['fdecay']) + optimizer = optim.AdamW(parameters=param_groups, learning_rate=self.config['flr']) + criterion = nn.MSELoss() + best_rmse_val, best_epoch = 999999, 0 + + self.start_time = time.time() + for epoch in range(self.config['epoch_num']): + self.train_one_epoch( + model=model, + train_loader=train_loader, + optimizer=optimizer, + epoch=epoch, + criterion=criterion, + ) + + rmse_val, mae_val, R2_val = self.evaluate( + model=model, + eval_loader=val_loader, + ) + + rmse_test, mae_test, R2_test = self.evaluate( + model=model, + eval_loader=test_loader, + ) + + if rmse_val < best_rmse_val: + best_epoch = epoch + best_rmse_val, best_mae_val, best_R2_val = rmse_val, mae_val, R2_val + best_rmse_test, best_mae_test, best_R2_test = rmse_test, mae_test, R2_test + star = '*** ' + else: star = '' + + log_new = star + "Epoch: %3d | Val RMSE: %.4f | Val MAE: %.4f | Val R2: %.4f | Test RMSE: %.4f | Test MAE: %.4f | Test R2: %.4f \n" \ + % (epoch, rmse_val, mae_val, R2_val, rmse_test, mae_test, R2_test) + self.log = self.log + log_new + '\n' + print(log_new) + + log_new = "Best Epoch: %3d | Best Val RMSE: %.4f | Best Val MAE: %.4f | Best Val R2: %.4f | Best Test RMSE: %.4f | Best Test MAE: %.4f | Best Test R2: %.4f " \ + % (best_epoch, best_rmse_val, best_mae_val, best_R2_val, best_rmse_test, best_mae_test, best_R2_test) + self.log = self.log + log_new + '\n' + print(log_new) + + + + def train_one_epoch(self, model, train_loader, optimizer, epoch, criterion): + model.train() + optimizer.clear_grad() + + for step, data in enumerate(train_loader): + ( + label, + poi_name_token_ids, + attn_mask_poi, + word_level_pos_ids, + poi_level_pos_ids, + grid_level_pos_ids, + poi_cate_ids, + img + ) = data + + label = label.cast('float32') + + poi_data = { + 'poi_name_token_ids': poi_name_token_ids, + 'attn_mask_poi': attn_mask_poi, + 'word_level_pos_ids': word_level_pos_ids, + 'poi_level_pos_ids': poi_level_pos_ids, + 'grid_level_pos_ids': grid_level_pos_ids, + 'poi_cate_ids': poi_cate_ids, + } + + img_data = { + 'img': img, + } + + if step % self.config['accum_iter'] == 0: + adjust_learning_rate( + optimizer=optimizer, + epoch=step / len(train_loader) + epoch, + warmup_epochs=self.config['warmup_epochs'], + epoch_num=self.config['epoch_num'], + peak_lr=self.config['flr'], + min_lr=self.config['min_lr']) + + pred = model(poi_data=poi_data, img_data=img_data) + pred = pred.squeeze(-1) + loss = criterion(pred, label) + + loss_value = loss.item() + + loss = loss / self.config['accum_iter'] + loss.backward() + + if (step + 1) % self.config['accum_iter'] == 0: + optimizer.step() + optimizer.clear_grad() + + paddle.device.cuda.synchronize() + total_time = time.time() - self.start_time + total_time = str(datetime.timedelta(seconds=int(total_time))) + + if step % config['logging_step'] == 0: + log_new = "Epoch: %3d | Step: %4d | Train loss: %9.4f | Time: %s" % (epoch, step, loss_value, total_time) + self.log = self.log + log_new + '\n' + print(log_new) + + + @paddle.no_grad() + def evaluate(self, model, eval_loader): + eval_pred_list, eval_label_list = [], [] + model.eval() + for _, data in enumerate(eval_loader()): + ( + label, + poi_name_token_ids, + attn_mask_poi, + word_level_pos_ids, + poi_level_pos_ids, + grid_level_pos_ids, + poi_cate_ids, + img + ) = data + + eval_label_list += label.tolist() + + poi_data = { + 'poi_name_token_ids': poi_name_token_ids, + 'attn_mask_poi': attn_mask_poi, + 'word_level_pos_ids': word_level_pos_ids, + 'poi_level_pos_ids': poi_level_pos_ids, + 'grid_level_pos_ids': grid_level_pos_ids, + 'poi_cate_ids': poi_cate_ids, + } + + img_data = { + 'img': img, + } + + pred = model(poi_data=poi_data, img_data=img_data) + pred = pred.squeeze(-1) + eval_pred_list += pred.cpu().tolist() + + rmse_eval = np.sqrt(mean_squared_error(eval_label_list, eval_pred_list)) + mae_eval = mean_absolute_error(eval_label_list, eval_pred_list) + R2_eval = r2_score(eval_label_list, eval_pred_list) + return rmse_eval, mae_eval, R2_eval + + + def Train(self): + seed_setup(self.config['seed']) + self.log = str(self.config) + '\n------------------- start training ----------------------\n' + self.train() + self.write_log() + + + def write_log(self): + log_output_path = self.log_path + self.param_info + if os.path.exists(log_output_path): + os.system('rm ' + log_output_path) + with open(log_output_path, 'w') as f: + f.write(self.log) + + +if __name__ == '__main__': + pass + parser = argparse.ArgumentParser() + parser.add_argument('--city', type=str, default=None) + parser.add_argument('--checkpoint', type=int, default=None) + parser.add_argument('--agg', type=str, default=None) + parser.add_argument('--fdrop', type=float, default=None) + parser.add_argument('--fbatch_size', type=int, default=None) + parser.add_argument('--epoch_num', type=int, default=None) + parser.add_argument('--warmup_epochs', type=int, default=None) + parser.add_argument('--flr', type=float, default=None) + parser.add_argument('--fdecay', type=float, default=None) + parser.add_argument('--min_lr', type=float, default=None) + parser.add_argument('--accum_iter', type=int, default=None) + parser.add_argument('--seed', type=int, default=None) + parser.add_argument('--logging_step', type=int, default=10) + args = parser.parse_args() + + for k, v in vars(args).items(): + config[k] = v + + config['hidden_dropout_prob_pretrain'] = config['hidden_dropout_prob'] + config['hidden_dropout_prob'] = config['fdrop'] + config['attention_probs_dropout_prob_pretrain'] = config['attention_probs_dropout_prob'] + config['attention_probs_dropout_prob'] = config['fdrop'] + + for k, v in config.items(): + print('{}: '.format(k), v, type(v)) + + os.environ['TOKENIZERS_PARALLELISM'] = 'false' + + Trainer_ = Trainer(config=config) + Trainer_.Train() \ No newline at end of file diff --git a/research/ReFound/code/finetune_pop.py b/research/ReFound/code/finetune_pop.py new file mode 100644 index 0000000..d98737b --- /dev/null +++ b/research/ReFound/code/finetune_pop.py @@ -0,0 +1,256 @@ +# -*- coding: utf-8 -*- + +import os +import json +import PIL.Image as pil +import random +import time +import datetime +import numpy as np +import paddle +import paddle.nn as nn +from paddle.io import DataLoader +import paddle.optimizer as optim +from sklearn.metrics import roc_curve, auc, precision_recall_curve +from sklearn.metrics import mean_squared_error, mean_absolute_error, r2_score +from collections import OrderedDict +import argparse +from utils import * +from ft_dataset_pop import * +from models import FinetunePopulationPrediction +from configuration import config + +class Trainer(object): + def __init__(self, config): + super(Trainer, self).__init__() + self.config = config + file_path = os.path.abspath(__file__) + self.bert_chinese_path = os.path.dirname(file_path) + '/../bert-base-chinese/' + self.log_path = os.path.dirname(file_path) + '/../log/POP/{c}/'.format(c=config['city']) + self.pretrain_state_path = os.path.dirname(file_path) + '/../checkpoint/' + + self.param_info = '{agg}_drop{fdrop}_bs{fbs}x{ac}_lr{flr}_dc{fdc}_ep{ep}.{warm}_seed{sd}'.format( + agg=config['agg'], fdrop=config['fdrop'], fbs=config['fbatch_size'], flr=config['flr'], fdc=config['fdecay'], + ac=config['accum_iter'], ep=config['epoch_num'], warm=config['warmup_epochs'], sd=config['seed'] + ) + + def get_dataloader(self): + train_dataset = FT_Dataset_POP(config=self.config, dataset_type='train') + val_dataset = FT_Dataset_POP(config=self.config, dataset_type='val') + test_dataset = FT_Dataset_POP(config=self.config, dataset_type='test') + train_loader = DataLoader(dataset=train_dataset, batch_size=self.config['fbatch_size'], shuffle=True, num_workers=1) + val_loader = DataLoader(dataset=val_dataset, batch_size=self.config['fbatch_size'], shuffle=False, num_workers=1) + test_loader = DataLoader(dataset=test_dataset, batch_size=self.config['fbatch_size'], shuffle=False, num_workers=1) + print(len(train_dataset), len(train_loader), len(val_dataset), len(val_loader), len(test_dataset), len(test_loader)) + return train_loader, val_loader, test_loader + + + def load_model(self): + model = FinetunePopulationPrediction(config=self.config) + checkpoint_path = self.pretrain_state_path + 'checkpoint-{}.pdparams'.format(self.config['checkpoint']) + print("checkpoint_path:\n" + checkpoint_path) + pretrain_state = paddle.load(checkpoint_path) + model.set_state_dict(pretrain_state) + return model + + + def train(self): + train_loader, val_loader, test_loader = self.get_dataloader() + model = self.load_model() + + param_groups = param_groups_lrd(model=model, weight_decay=self.config['fdecay']) + optimizer = optim.AdamW(parameters=param_groups, learning_rate=self.config['flr']) + criterion = nn.MSELoss() + best_rmse_val, best_epoch = 999999, 0 + + self.start_time = time.time() + for epoch in range(self.config['epoch_num']): + self.train_one_epoch( + model=model, + train_loader=train_loader, + optimizer=optimizer, + epoch=epoch, + criterion=criterion, + ) + + rmse_val, mae_val, R2_val = self.evaluate( + model=model, + eval_loader=val_loader, + ) + + rmse_test, mae_test, R2_test = self.evaluate( + model=model, + eval_loader=test_loader, + ) + + if rmse_val < best_rmse_val: + best_epoch = epoch + best_rmse_val, best_mae_val, best_R2_val = rmse_val, mae_val, R2_val + best_rmse_test, best_mae_test, best_R2_test = rmse_test, mae_test, R2_test + star = '*** ' + else: star = '' + + log_new = star + "Epoch: %3d | Val RMSE: %.4f | Val MAE: %.4f | Val R2: %.4f | Test RMSE: %.4f | Test MAE: %.4f | Test R2: %.4f \n" \ + % (epoch, rmse_val, mae_val, R2_val, rmse_test, mae_test, R2_test) + self.log = self.log + log_new + '\n' + print(log_new) + + log_new = "Best Epoch: %3d | Best Val RMSE: %.4f | Best Val MAE: %.4f | Best Val R2: %.4f | Best Test RMSE: %.4f | Best Test MAE: %.4f | Best Test R2: %.4f " \ + % (best_epoch, best_rmse_val, best_mae_val, best_R2_val, best_rmse_test, best_mae_test, best_R2_test) + self.log = self.log + log_new + '\n' + print(log_new) + + + + def train_one_epoch(self, model, train_loader, optimizer, epoch, criterion): + model.train() + optimizer.clear_grad() + + for step, data in enumerate(train_loader): + ( + label, + poi_name_token_ids, + attn_mask_poi, + word_level_pos_ids, + poi_level_pos_ids, + grid_level_pos_ids, + poi_cate_ids, + img + ) = data + + label = label.cast('float32') + + poi_data = { + 'poi_name_token_ids': poi_name_token_ids, + 'attn_mask_poi': attn_mask_poi, + 'word_level_pos_ids': word_level_pos_ids, + 'poi_level_pos_ids': poi_level_pos_ids, + 'grid_level_pos_ids': grid_level_pos_ids, + 'poi_cate_ids': poi_cate_ids, + } + + img_data = { + 'img': img, + } + + if step % self.config['accum_iter'] == 0: + adjust_learning_rate( + optimizer=optimizer, + epoch=step / len(train_loader) + epoch, + warmup_epochs=self.config['warmup_epochs'], + epoch_num=self.config['epoch_num'], + peak_lr=self.config['flr'], + min_lr=self.config['min_lr']) + + pred = model(poi_data=poi_data, img_data=img_data) + pred = pred.squeeze(-1) + loss = criterion(pred, label) + + loss_value = loss.item() + + loss = loss / self.config['accum_iter'] + loss.backward() + + if (step + 1) % self.config['accum_iter'] == 0: + optimizer.step() + optimizer.clear_grad() + + paddle.device.cuda.synchronize() + total_time = time.time() - self.start_time + total_time = str(datetime.timedelta(seconds=int(total_time))) + + if step % config['logging_step'] == 0: + log_new = "Epoch: %3d | Step: %4d | Train loss: %9.4f | Time: %s" % (epoch, step, loss_value, total_time) + self.log = self.log + log_new + '\n' + print(log_new) + + + @paddle.no_grad() + def evaluate(self, model, eval_loader): + eval_pred_list, eval_label_list = [], [] + model.eval() + for _, data in enumerate(eval_loader()): + ( + label, + poi_name_token_ids, + attn_mask_poi, + word_level_pos_ids, + poi_level_pos_ids, + grid_level_pos_ids, + poi_cate_ids, + img + ) = data + + eval_label_list += label.tolist() + + poi_data = { + 'poi_name_token_ids': poi_name_token_ids, + 'attn_mask_poi': attn_mask_poi, + 'word_level_pos_ids': word_level_pos_ids, + 'poi_level_pos_ids': poi_level_pos_ids, + 'grid_level_pos_ids': grid_level_pos_ids, + 'poi_cate_ids': poi_cate_ids, + } + + img_data = { + 'img': img, + } + + pred = model(poi_data=poi_data, img_data=img_data) + pred = pred.squeeze(-1) + eval_pred_list += pred.cpu().tolist() + + rmse_eval = np.sqrt(mean_squared_error(eval_label_list, eval_pred_list)) + mae_eval = mean_absolute_error(eval_label_list, eval_pred_list) + R2_eval = r2_score(eval_label_list, eval_pred_list) + return rmse_eval, mae_eval, R2_eval + + + def Train(self): + seed_setup(self.config['seed']) + self.log = str(self.config) + '\n------------------- start training ----------------------\n' + self.train() + self.write_log() + + + def write_log(self): + log_output_path = self.log_path + self.param_info + if os.path.exists(log_output_path): + os.system('rm ' + log_output_path) + with open(log_output_path, 'w') as f: + f.write(self.log) + + +if __name__ == '__main__': + pass + parser = argparse.ArgumentParser() + parser.add_argument('--city', type=str, default=None) + parser.add_argument('--checkpoint', type=int, default=None) + parser.add_argument('--agg', type=str, default=None) + parser.add_argument('--fdrop', type=float, default=None) + parser.add_argument('--fbatch_size', type=int, default=None) + parser.add_argument('--epoch_num', type=int, default=None) + parser.add_argument('--warmup_epochs', type=int, default=None) + parser.add_argument('--flr', type=float, default=None) + parser.add_argument('--fdecay', type=float, default=None) + parser.add_argument('--min_lr', type=float, default=None) + parser.add_argument('--accum_iter', type=int, default=None) + parser.add_argument('--seed', type=int, default=None) + parser.add_argument('--logging_step', type=int, default=10) + args = parser.parse_args() + + for k, v in vars(args).items(): + config[k] = v + + config['hidden_dropout_prob_pretrain'] = config['hidden_dropout_prob'] + config['hidden_dropout_prob'] = config['fdrop'] + config['attention_probs_dropout_prob_pretrain'] = config['attention_probs_dropout_prob'] + config['attention_probs_dropout_prob'] = config['fdrop'] + + for k, v in config.items(): + print('{}: '.format(k), v, type(v)) + + os.environ['TOKENIZERS_PARALLELISM'] = 'false' + + Trainer_ = Trainer(config=config) + Trainer_.Train() \ No newline at end of file diff --git a/research/ReFound/code/finetune_uvd.py b/research/ReFound/code/finetune_uvd.py new file mode 100644 index 0000000..74607ad --- /dev/null +++ b/research/ReFound/code/finetune_uvd.py @@ -0,0 +1,296 @@ +# -*- coding: utf-8 -*- + +import os +import json +import PIL.Image as pil +import random +import time +import datetime +import numpy as np +import paddle +import paddle.nn as nn +from paddle.io import DataLoader +import paddle.optimizer as optim +from sklearn.metrics import roc_curve, auc, precision_recall_curve +from sklearn.metrics import mean_squared_error, mean_absolute_error, r2_score +from collections import OrderedDict +import argparse +from utils import * +from ft_dataset_uvd import * +from models import FinetuneUrbanVillageDetection +from configuration import config + + +class Trainer(object): + def __init__(self, config): + super(Trainer, self).__init__() + self.config = config + file_path = os.path.abspath(__file__) + self.bert_chinese_path = os.path.dirname(file_path) + '/../bert-base-chinese/' + self.log_path = os.path.dirname(file_path) + '/../log/UVD/{c}/'.format(c=config['city']) + self.model_path = os.path.dirname(file_path) + '/../model/UVD/{c}/'.format(c=config['city']) + self.prob_path = os.path.dirname(file_path) + '/../prob_label/UVD/{c}/'.format(c=config['city']) + self.pretrain_state_path = os.path.dirname(file_path) + '/../checkpoint/' + + self.param_info = '{agg}_drop{fdrop}_bs{fbs}x{ac}_lr{flr}_dc{fdc}_ep{ep}.{warm}_seed{sd}'.format( + agg=config['agg'], fdrop=config['fdrop'], fbs=config['fbatch_size'], flr=config['flr'], fdc=config['fdecay'], + ac=config['accum_iter'], ep=config['epoch_num'], warm=config['warmup_epochs'], sd=config['seed'] + ) + + + def get_dataloader(self): + train_dataset = FT_Dataset_UVD(config=self.config, dataset_type='train') + val_dataset = FT_Dataset_UVD(config=self.config, dataset_type='val') + test_dataset = FT_Dataset_UVD(config=self.config, dataset_type='test') + train_loader = DataLoader(dataset=train_dataset, batch_size=self.config['fbatch_size'], shuffle=True, num_workers=1) + train_loader_for_eval = DataLoader(dataset=train_dataset, batch_size=self.config['fbatch_size'], shuffle=False, num_workers=1) + val_loader = DataLoader(dataset=val_dataset, batch_size=self.config['fbatch_size'], shuffle=False, num_workers=1) + test_loader = DataLoader(dataset=test_dataset, batch_size=self.config['fbatch_size'], shuffle=False, num_workers=1) + print(len(train_dataset), len(train_loader), len(val_dataset), len(val_loader), len(test_dataset), len(test_loader)) + return train_dataset, val_dataset, test_dataset, train_loader, train_loader_for_eval, val_loader, test_loader + + + def load_model(self): + model = FinetuneUrbanVillageDetection(config=self.config) + checkpoint_path = self.pretrain_state_path + 'checkpoint-{}.pdparams'.format(self.config['checkpoint']) + print("checkpoint_path:\n" + checkpoint_path) + pretrain_state = paddle.load(checkpoint_path) + model.set_state_dict(pretrain_state) + return model + + + def train(self, save_epoch=-1): + train_dataset, val_dataset, test_dataset, train_loader, train_loader_for_eval, val_loader, test_loader = self.get_dataloader() + model = self.load_model() + + param_groups = param_groups_lrd(model=model, weight_decay=self.config['fdecay']) + optimizer = optim.AdamW(parameters=param_groups, learning_rate=self.config['flr']) + criterion = nn.BCELoss() + best_AUC_val, best_epoch = 0, 0 + + prob_label_dict = {'prob': {}, 'label': {}} + prob_label_dict['label']['train'] = train_dataset.label_list + prob_label_dict['label']['val'] = val_dataset.label_list + prob_label_dict['label']['test'] = test_dataset.label_list + + self.start_time = time.time() + for epoch in range(self.config['epoch_num']): + self.train_one_epoch( + model=model, + train_loader=train_loader, + optimizer=optimizer, + epoch=epoch, + criterion=criterion, + ) + + AUC_train, train_prob_list = self.evaluate( + model=model, + eval_loader=train_loader_for_eval, + ) + + AUC_val, val_prob_list = self.evaluate( + model=model, + eval_loader=val_loader, + ) + + AUC_test, test_prob_list = self.evaluate( + model=model, + eval_loader=test_loader, + ) + + if AUC_val > best_AUC_val: + best_AUC_val = AUC_val + best_epoch = epoch + best_AUC_test = AUC_test + star = '*** ' + + prob_label_dict['prob']['train'] = train_prob_list + prob_label_dict['prob']['val'] = val_prob_list + prob_label_dict['prob']['test'] = test_prob_list + + else: star = '' + + log_new = star + "Epoch: %3d | Val AUC: %.4f | Test AUC: %.4f \n" % (epoch, AUC_val, AUC_test) + self.log = self.log + log_new + '\n' + print(log_new) + + if epoch == save_epoch: + self.save_model(model) + break + + log_new = "Best Epoch: %3d | Best Val AUC: %.4f | Best Test AUC: %.4f" % (best_epoch, best_AUC_val, best_AUC_test) + self.log = self.log + log_new + '\n' + print(log_new) + + self.save_prob_label(prob_label_dict=prob_label_dict) + + return best_epoch + + + def train_one_epoch(self, model, train_loader, optimizer, epoch, criterion): + model.train() + optimizer.clear_grad() + + for step, data in enumerate(train_loader()): + ( + label, + poi_name_token_ids, + attn_mask_poi, + word_level_pos_ids, + poi_level_pos_ids, + grid_level_pos_ids, + poi_cate_ids, + img + ) = data + + label = label.cast('float32') + + poi_data = { + 'poi_name_token_ids': poi_name_token_ids, + 'attn_mask_poi': attn_mask_poi, + 'word_level_pos_ids': word_level_pos_ids, + 'poi_level_pos_ids': poi_level_pos_ids, + 'grid_level_pos_ids': grid_level_pos_ids, + 'poi_cate_ids': poi_cate_ids, + } + + img_data = { + 'img': img, + } + + if step % self.config['accum_iter'] == 0: + adjust_learning_rate( + optimizer=optimizer, + epoch=step / len(train_loader) + epoch, + warmup_epochs=self.config['warmup_epochs'], + epoch_num=self.config['epoch_num'], + peak_lr=self.config['flr'], + min_lr=self.config['min_lr']) + + prob = model(poi_data=poi_data, img_data=img_data) + prob = prob.squeeze(-1) + loss = criterion(prob, label) + + loss_value = loss.item() + + loss = loss / self.config['accum_iter'] + loss.backward() + + if (step + 1) % self.config['accum_iter'] == 0: + optimizer.step() + optimizer.clear_grad() + + paddle.device.cuda.synchronize() + total_time = time.time() - self.start_time + total_time = str(datetime.timedelta(seconds=int(total_time))) + + if step % config['logging_step'] == 0: + log_new = "Epoch: %3d | Step: %4d | Train loss: %7.4f | Time: %s" % (epoch, step, loss_value, total_time) + self.log = self.log + log_new + '\n' + print(log_new) + + + @paddle.no_grad() + def evaluate(self, model, eval_loader): + eval_prob_list, eval_label_list = [], [] + model.eval() + for _, data in enumerate(eval_loader()): + ( + label, + poi_name_token_ids, + attn_mask_poi, + word_level_pos_ids, + poi_level_pos_ids, + grid_level_pos_ids, + poi_cate_ids, + img + ) = data + + eval_label_list += label.tolist() + + poi_data = { + 'poi_name_token_ids': poi_name_token_ids, + 'attn_mask_poi': attn_mask_poi, + 'word_level_pos_ids': word_level_pos_ids, + 'poi_level_pos_ids': poi_level_pos_ids, + 'grid_level_pos_ids': grid_level_pos_ids, + 'poi_cate_ids': poi_cate_ids, + } + + img_data = { + 'img': img, + } + + prob = model(poi_data=poi_data, img_data=img_data) + prob = prob.squeeze(-1) + eval_prob_list += prob.cpu().tolist() + + fpr, tpr, _ = roc_curve(eval_label_list, eval_prob_list, pos_label=1) + AUC_eval = auc(fpr, tpr) + return AUC_eval, eval_prob_list + + + def Train(self): + # select best epoch based on validation set + seed_setup(self.config['seed']) + self.log = str(self.config) + '\n------------------- start training ----------------------\n' + best_epoch = self.train() + self.write_log() + + # # re-train model to save on the best epoch + # seed_setup(self.config['seed']) + # self.train(save_epoch=best_epoch) + + + def write_log(self): + log_output_path = self.log_path + self.param_info + if os.path.exists(log_output_path): + os.system('rm ' + log_output_path) + with open(log_output_path, 'w') as f: + f.write(self.log) + + + def save_model(self, model): + save_dir = self.model_path + self.param_info + paddle.save(model.state_dict(), save_dir) + + + def save_prob_label(self, prob_label_dict): + prob_label_path = self.prob_path + self.param_info + with open(prob_label_path, 'w') as f: + f.write(str(prob_label_dict)) + + +if __name__ == '__main__': + pass + parser = argparse.ArgumentParser() + parser.add_argument('--city', type=str, default=None) + parser.add_argument('--checkpoint', type=int, default=None) + parser.add_argument('--agg', type=str, default=None) + parser.add_argument('--fdrop', type=float, default=None) + parser.add_argument('--fbatch_size', type=int, default=None) + parser.add_argument('--epoch_num', type=int, default=None) + parser.add_argument('--warmup_epochs', type=int, default=None) + parser.add_argument('--flr', type=float, default=None) + parser.add_argument('--fdecay', type=float, default=None) + parser.add_argument('--min_lr', type=float, default=None) + parser.add_argument('--accum_iter', type=int, default=None) + parser.add_argument('--seed', type=int, default=None) + parser.add_argument('--logging_step', type=int, default=1) + args = parser.parse_args() + + for k, v in vars(args).items(): + config[k] = v + + config['hidden_dropout_prob_pretrain'] = config['hidden_dropout_prob'] + config['hidden_dropout_prob'] = config['fdrop'] + config['attention_probs_dropout_prob_pretrain'] = config['attention_probs_dropout_prob'] + config['attention_probs_dropout_prob'] = config['fdrop'] + + for k, v in config.items(): + print('{}: '.format(k), v, type(v)) + + os.environ['TOKENIZERS_PARALLELISM'] = 'false' + + Trainer_ = Trainer(config=config) + Trainer_.Train() \ No newline at end of file diff --git a/research/ReFound/code/ft_dataset_cap.py b/research/ReFound/code/ft_dataset_cap.py new file mode 100644 index 0000000..757b9dc --- /dev/null +++ b/research/ReFound/code/ft_dataset_cap.py @@ -0,0 +1,193 @@ +# -*- coding: utf-8 -*- + +import os +import json +import PIL.Image as pil +import random +from transformers import AutoTokenizer +import paddle +from paddle.io import Dataset +from collections import OrderedDict, Counter +from utils import * + + + + +class FT_Dataset_CAP(Dataset): + def __init__(self, config=None, dataset_type=None): + super().__init__() + self.config = config + file_path = os.path.abspath(__file__) + self.root_path = os.path.dirname(file_path) + '/../' + self.id_of_region_path = self.root_path + 'data/cap_{c}_id_of_region_{dt}'.format(c=config['city'], dt=dataset_type) + self.label_path = self.root_path + 'data/cap_{c}_label_{dt}'.format(c=config['city'], dt=dataset_type) + self.region_coord_path = self.root_path + 'data/cap_{c}_region_coord_{dt}'.format(c=config['city'], dt=dataset_type) + self.poi_sortby_zorder_path = self.root_path + 'data/cap_{c}_poi_sort_by_zorder_{dt}'.format(c=config['city'], dt=dataset_type) + self.poi_cate_vocab_path = self.root_path + 'data/poi_cate_vocab' + self.bert_chinese_path = self.root_path + 'bert-base-chinese/' + self.dataset_type = dataset_type + self.img_path = self.root_path + 'data/satellite_img/{c}/'.format(c=config['city']) + + self.id_of_region = [] + with open(self.id_of_region_path, 'r') as f: + for line in f: + line = line.strip('\n') + self.id_of_region.append(line) + + label_list = [] + with open(self.label_path, 'r') as f: + for line in f: + line = eval(line.strip('\n')) + label_list.append(line) + + self.label_list = [float(label) for label in label_list] + + + image_height, image_width = pair(config['image_size']) + patch_height, patch_width = pair(config['patch_size']) + assert image_height % patch_height == 0 and image_width % patch_width == 0, \ + 'Image dimensions must be divisible by the patch size.' + self.num_patch = (image_height // patch_height) * (image_width // patch_width) + + self.max_len_token = config['max_len_token'] + self.max_len_poi = config['max_len_poi'] + self.num_grid_x = config['num_grid_x'] + self.num_grid_y = config['num_grid_y'] + self.num_grid = self.num_grid_x * self.num_grid_y + + + self.poi_zorder_list = [] + with open(self.poi_sortby_zorder_path, 'r') as f: + for line in f: + line = eval(line.strip('\n')) + self.poi_zorder_list.append(line[1]) + + + self.tokenizer = AutoTokenizer.from_pretrained(self.bert_chinese_path) + self.cls_token_id = self.tokenizer.cls_token_id + self.pad_token_id = self.tokenizer.pad_token_id + self.mask_token_id = self.tokenizer.mask_token_id + self.sep_token_id = self.tokenizer.sep_token_id + + + self.region_coord_list = [] + with open(self.region_coord_path, 'r') as f: + for line in f: + line = eval(line.strip('\n')) + self.region_coord_list.append(line) + + self.poi_cate_vocab = {} + with open(self.poi_cate_vocab_path, 'r') as f: + for line in f: + line = line.strip('\n').split('\t') + cate, cate_id = line + self.poi_cate_vocab[cate] = int(cate_id) + + + + mean = (0.38247773301793103, 0.41271937512626544, 0.3403674447902101) + std = (0.17229526604561052, 0.14788625733896113, 0.16325427643628246) + self.img_trans = paddle.vision.transforms.Normalize(mean, std) + + + + def tokenize_poi_data(self, index): + poi_list = self.poi_zorder_list[index] + region_coord = self.region_coord_list[index] + + if poi_list is None: + token_id_seq = [self.cls_token_id] + [self.pad_token_id] * (self.max_len_token - 1) + attn_mask_seq = [1] + [0] * (self.max_len_token - 1) + word_level_pos_id_seq = [0]+ [i % self.max_len_token for i in range(self.max_len_token - 1)] + poi_level_pos_id_seq = [0] + [1 + int(i / self.max_len_token) for i in range(self.max_len_token - 1)] + grid_level_pos_id_seq = [0] + [self.num_grid + 1] * (self.max_len_token - 1) + offset = [1] + poi_cate_id_seq = [self.poi_cate_vocab['PAD']] * self.max_len_token + + + else: + cur_len, num_poi = 1, 1 + token_id_seq, attn_mask_seq = [self.cls_token_id], [1] + word_level_pos_id_seq, poi_level_pos_id_seq, grid_level_pos_id_seq = [0], [0], [0] + offset = [cur_len] + poi_cate_id_seq = [self.poi_cate_vocab['PAD']] + + + for poi in poi_list: + poi_name, _, poi_cate_id, poi_x, poi_y = poi + + poi_name = poi_name.lower() + tokenize_output = self.tokenizer(poi_name, add_special_tokens=False) + + token_id = tokenize_output['input_ids'] + attn_mask = tokenize_output['attention_mask'] + + token_id.append(self.sep_token_id) # add sep token behind each poi + attn_mask.append(1) + + word_level_pos_id = list(range(len(token_id))) + + poi_level_pos_id = [num_poi] * len(token_id) + + grid_x = int(self.num_grid_x * (poi_x - region_coord[0]) / (region_coord[2] - region_coord[0])) + grid_y = int(self.num_grid_y * (region_coord[1] - poi_y) / (region_coord[1] - region_coord[3])) + grid_x, grid_y = min(grid_x, self.num_grid_x - 1), min(grid_y, self.num_grid_y - 1) + grid_id = self.num_grid_x * grid_y + grid_x + 1 + grid_level_pos_id = [grid_id] * len(token_id) + + poi_cate_id = [poi_cate_id] * len(token_id) + + cur_len += len(token_id) + if cur_len <= self.max_len_token: + offset.append(cur_len) + token_id_seq.extend(token_id) + attn_mask_seq.extend(attn_mask) + word_level_pos_id_seq.extend(word_level_pos_id) + poi_level_pos_id_seq.extend(poi_level_pos_id) + grid_level_pos_id_seq.extend(grid_level_pos_id) + poi_cate_id_seq.extend(poi_cate_id) + num_poi += 1 + + else: + break + + ## padding + padding_len = self.max_len_token - offset[-1] + if padding_len > 0: + token_id_seq.extend([self.pad_token_id] * padding_len) + attn_mask_seq.extend([0] * padding_len) + word_level_pos_id_seq.extend([i % self.max_len_token for i in range(padding_len)]) + poi_level_pos_id_seq.extend([num_poi + int(i / self.max_len_poi) for i in range(padding_len)]) + grid_level_pos_id_seq.extend([self.num_grid + 1] * padding_len) + poi_cate_id_seq.extend([self.poi_cate_vocab['PAD']] * padding_len) + + return token_id_seq, attn_mask_seq, word_level_pos_id_seq, poi_level_pos_id_seq, grid_level_pos_id_seq, poi_cate_id_seq + + + + def __getitem__(self, index): + id = self.id_of_region[index] + label = self.label_list[index] + + poi_name_token_ids, attn_mask_poi, word_level_pos_ids, poi_level_pos_ids, grid_level_pos_ids, poi_cate_ids = self.tokenize_poi_data(index) + + img = pil_loader(self.img_path + '{id}.png'.format(id=id)) + img = np.array(img).astype('float32') / 255.0 + img = img.transpose(2, 0, 1) + img = self.img_trans(img) + + poi_name_token_ids = np.array(poi_name_token_ids) + attn_mask_poi = np.array(attn_mask_poi) + word_level_pos_ids = np.array(word_level_pos_ids) + poi_level_pos_ids = np.array(poi_level_pos_ids) + grid_level_pos_ids = np.array(grid_level_pos_ids) + poi_cate_ids = np.array(poi_cate_ids) + + return label, poi_name_token_ids, attn_mask_poi, word_level_pos_ids, poi_level_pos_ids, grid_level_pos_ids, poi_cate_ids, img + + + def __len__(self): + return len(self.id_of_region) + + + \ No newline at end of file diff --git a/research/ReFound/code/ft_dataset_pop.py b/research/ReFound/code/ft_dataset_pop.py new file mode 100644 index 0000000..662a6ee --- /dev/null +++ b/research/ReFound/code/ft_dataset_pop.py @@ -0,0 +1,193 @@ +# -*- coding: utf-8 -*- + +import os +import json +import PIL.Image as pil +import random +from transformers import AutoTokenizer +import paddle +from paddle.io import Dataset +from collections import OrderedDict, Counter +from utils import * + + + + +class FT_Dataset_POP(Dataset): + def __init__(self, config=None, dataset_type=None): + super().__init__() + self.config = config + file_path = os.path.abspath(__file__) + self.root_path = os.path.dirname(file_path) + '/../' + self.id_of_region_path = self.root_path + 'data/pop_{c}_id_of_region_{dt}'.format(c=config['city'], dt=dataset_type) + self.label_path = self.root_path + 'data/pop_{c}_label_{dt}'.format(c=config['city'], dt=dataset_type) + self.region_coord_path = self.root_path + 'data/pop_{c}_region_coord_{dt}'.format(c=config['city'], dt=dataset_type) + self.poi_sortby_zorder_path = self.root_path + 'data/pop_{c}_poi_sort_by_zorder_{dt}'.format(c=config['city'], dt=dataset_type) + self.poi_cate_vocab_path = self.root_path + 'data/poi_cate_vocab' + self.bert_chinese_path = self.root_path + 'bert-base-chinese/' + self.dataset_type = dataset_type + self.img_path = self.root_path + 'data/satellite_img/{c}/'.format(c=config['city']) + + self.id_of_region = [] + with open(self.id_of_region_path, 'r') as f: + for line in f: + line = line.strip('\n') + self.id_of_region.append(line) + + label_list = [] + with open(self.label_path, 'r') as f: + for line in f: + line = eval(line.strip('\n')) + label_list.append(line) + + self.label_list = [float(label) for label in label_list] + + + image_height, image_width = pair(config['image_size']) + patch_height, patch_width = pair(config['patch_size']) + assert image_height % patch_height == 0 and image_width % patch_width == 0, \ + 'Image dimensions must be divisible by the patch size.' + self.num_patch = (image_height // patch_height) * (image_width // patch_width) + + self.max_len_token = config['max_len_token'] + self.max_len_poi = config['max_len_poi'] + self.num_grid_x = config['num_grid_x'] + self.num_grid_y = config['num_grid_y'] + self.num_grid = self.num_grid_x * self.num_grid_y + + + self.poi_zorder_list = [] + with open(self.poi_sortby_zorder_path, 'r') as f: + for line in f: + line = eval(line.strip('\n')) + self.poi_zorder_list.append(line[1]) + + + self.tokenizer = AutoTokenizer.from_pretrained(self.bert_chinese_path) + self.cls_token_id = self.tokenizer.cls_token_id + self.pad_token_id = self.tokenizer.pad_token_id + self.mask_token_id = self.tokenizer.mask_token_id + self.sep_token_id = self.tokenizer.sep_token_id + + + self.region_coord_list = [] + with open(self.region_coord_path, 'r') as f: + for line in f: + line = eval(line.strip('\n')) + self.region_coord_list.append(line) + + self.poi_cate_vocab = {} + with open(self.poi_cate_vocab_path, 'r') as f: + for line in f: + line = line.strip('\n').split('\t') + cate, cate_id = line + self.poi_cate_vocab[cate] = int(cate_id) + + + + mean = (0.38247773301793103, 0.41271937512626544, 0.3403674447902101) + std = (0.17229526604561052, 0.14788625733896113, 0.16325427643628246) + self.img_trans = paddle.vision.transforms.Normalize(mean, std) + + + + def tokenize_poi_data(self, index): + poi_list = self.poi_zorder_list[index] + region_coord = self.region_coord_list[index] + + if poi_list is None: + token_id_seq = [self.cls_token_id] + [self.pad_token_id] * (self.max_len_token - 1) + attn_mask_seq = [1] + [0] * (self.max_len_token - 1) + word_level_pos_id_seq = [0]+ [i % self.max_len_token for i in range(self.max_len_token - 1)] + poi_level_pos_id_seq = [0] + [1 + int(i / self.max_len_token) for i in range(self.max_len_token - 1)] + grid_level_pos_id_seq = [0] + [self.num_grid + 1] * (self.max_len_token - 1) + offset = [1] + poi_cate_id_seq = [self.poi_cate_vocab['PAD']] * self.max_len_token + + + else: + cur_len, num_poi = 1, 1 + token_id_seq, attn_mask_seq = [self.cls_token_id], [1] + word_level_pos_id_seq, poi_level_pos_id_seq, grid_level_pos_id_seq = [0], [0], [0] + offset = [cur_len] + poi_cate_id_seq = [self.poi_cate_vocab['PAD']] + + + for poi in poi_list: + poi_name, _, poi_cate_id, poi_x, poi_y = poi + + poi_name = poi_name.lower() + tokenize_output = self.tokenizer(poi_name, add_special_tokens=False) + + token_id = tokenize_output['input_ids'] + attn_mask = tokenize_output['attention_mask'] + + token_id.append(self.sep_token_id) # add sep token behind each poi + attn_mask.append(1) + + word_level_pos_id = list(range(len(token_id))) + + poi_level_pos_id = [num_poi] * len(token_id) + + grid_x = int(self.num_grid_x * (poi_x - region_coord[0]) / (region_coord[2] - region_coord[0])) + grid_y = int(self.num_grid_y * (region_coord[1] - poi_y) / (region_coord[1] - region_coord[3])) + grid_x, grid_y = min(grid_x, self.num_grid_x - 1), min(grid_y, self.num_grid_y - 1) + grid_id = self.num_grid_x * grid_y + grid_x + 1 + grid_level_pos_id = [grid_id] * len(token_id) + + poi_cate_id = [poi_cate_id] * len(token_id) + + cur_len += len(token_id) + if cur_len <= self.max_len_token: + offset.append(cur_len) + token_id_seq.extend(token_id) + attn_mask_seq.extend(attn_mask) + word_level_pos_id_seq.extend(word_level_pos_id) + poi_level_pos_id_seq.extend(poi_level_pos_id) + grid_level_pos_id_seq.extend(grid_level_pos_id) + poi_cate_id_seq.extend(poi_cate_id) + num_poi += 1 + + else: + break + + ## padding + padding_len = self.max_len_token - offset[-1] + if padding_len > 0: + token_id_seq.extend([self.pad_token_id] * padding_len) + attn_mask_seq.extend([0] * padding_len) + word_level_pos_id_seq.extend([i % self.max_len_token for i in range(padding_len)]) + poi_level_pos_id_seq.extend([num_poi + int(i / self.max_len_poi) for i in range(padding_len)]) + grid_level_pos_id_seq.extend([self.num_grid + 1] * padding_len) + poi_cate_id_seq.extend([self.poi_cate_vocab['PAD']] * padding_len) + + return token_id_seq, attn_mask_seq, word_level_pos_id_seq, poi_level_pos_id_seq, grid_level_pos_id_seq, poi_cate_id_seq + + + + def __getitem__(self, index): + id = self.id_of_region[index] + label = self.label_list[index] + + poi_name_token_ids, attn_mask_poi, word_level_pos_ids, poi_level_pos_ids, grid_level_pos_ids, poi_cate_ids = self.tokenize_poi_data(index) + + img = pil_loader(self.img_path + '{id}.png'.format(id=id)) + img = np.array(img).astype('float32') / 255.0 + img = img.transpose(2, 0, 1) + img = self.img_trans(img) + + poi_name_token_ids = np.array(poi_name_token_ids) + attn_mask_poi = np.array(attn_mask_poi) + word_level_pos_ids = np.array(word_level_pos_ids) + poi_level_pos_ids = np.array(poi_level_pos_ids) + grid_level_pos_ids = np.array(grid_level_pos_ids) + poi_cate_ids = np.array(poi_cate_ids) + + return label, poi_name_token_ids, attn_mask_poi, word_level_pos_ids, poi_level_pos_ids, grid_level_pos_ids, poi_cate_ids, img + + + def __len__(self): + return len(self.id_of_region) + + + \ No newline at end of file diff --git a/research/ReFound/code/ft_dataset_uvd.py b/research/ReFound/code/ft_dataset_uvd.py new file mode 100644 index 0000000..43c72f1 --- /dev/null +++ b/research/ReFound/code/ft_dataset_uvd.py @@ -0,0 +1,190 @@ +# -*- coding: utf-8 -*- + +import os +import json +import PIL.Image as pil +import random +from transformers import AutoTokenizer +import paddle +from paddle.io import Dataset +from collections import OrderedDict, Counter +from utils import * + + + +class FT_Dataset_UVD(Dataset): + def __init__(self, config=None, dataset_type=None): + super().__init__() + self.config = config + file_path = os.path.abspath(__file__) + self.root_path = os.path.dirname(file_path) + '/../' + self.id_of_region_path = self.root_path + 'data/uvd_{c}_id_of_region_{dt}'.format(c=config['city'], dt=dataset_type) + self.label_path = self.root_path + 'data/uvd_{c}_label_{dt}'.format(c=config['city'], dt=dataset_type) + self.region_coord_path = self.root_path + 'data/uvd_{c}_region_coord_{dt}'.format(c=config['city'], dt=dataset_type) + self.poi_sortby_zorder_path = self.root_path + 'data/uvd_{c}_poi_sort_by_zorder_{dt}'.format(c=config['city'], dt=dataset_type) + self.poi_cate_vocab_path = self.root_path + 'data/poi_cate_vocab' + self.bert_chinese_path = self.root_path + 'bert-base-chinese/' + self.dataset_type = dataset_type + self.img_path = self.root_path + 'data/satellite_img/{c}/'.format(c=config['city']) + + + self.id_of_region = [] + with open(self.id_of_region_path, 'r') as f: + for line in f: + line = line.strip('\n') + self.id_of_region.append(line) + + label_list = [] + with open(self.label_path, 'r') as f: + for line in f: + line = eval(line.strip('\n')) + label_list.append(line) + + self.label_list = [float(label) for label in label_list] + + + image_height, image_width = pair(config['image_size']) + patch_height, patch_width = pair(config['patch_size']) + assert image_height % patch_height == 0 and image_width % patch_width == 0, \ + 'Image dimensions must be divisible by the patch size.' + self.num_patch = (image_height // patch_height) * (image_width // patch_width) + + self.max_len_token = config['max_len_token'] + self.max_len_poi = config['max_len_poi'] + self.num_grid_x = config['num_grid_x'] + self.num_grid_y = config['num_grid_y'] + self.num_grid = self.num_grid_x * self.num_grid_y + + + self.poi_zorder_list = [] + with open(self.poi_sortby_zorder_path, 'r') as f: + for line in f: + line = eval(line.strip('\n')) + self.poi_zorder_list.append(line[1]) + + + self.tokenizer = AutoTokenizer.from_pretrained(self.bert_chinese_path) + self.cls_token_id = self.tokenizer.cls_token_id + self.pad_token_id = self.tokenizer.pad_token_id + self.mask_token_id = self.tokenizer.mask_token_id + self.sep_token_id = self.tokenizer.sep_token_id + + + self.region_coord_list = [] + with open(self.region_coord_path, 'r') as f: + for line in f: + line = eval(line.strip('\n')) + self.region_coord_list.append(line) + + self.poi_cate_vocab = {} + with open(self.poi_cate_vocab_path, 'r') as f: + for line in f: + line = line.strip('\n').split('\t') + cate, cate_id = line + self.poi_cate_vocab[cate] = int(cate_id) + + + mean = (0.38247773301793103, 0.41271937512626544, 0.3403674447902101) + std = (0.17229526604561052, 0.14788625733896113, 0.16325427643628246) + self.img_trans = paddle.vision.transforms.Normalize(mean, std) + + + + def tokenize_poi_data(self, index): + poi_list = self.poi_zorder_list[index] + region_coord = self.region_coord_list[index] + + if poi_list is None: + token_id_seq = [self.cls_token_id] + [self.pad_token_id] * (self.max_len_token - 1) + attn_mask_seq = [1] + [0] * (self.max_len_token - 1) + word_level_pos_id_seq = [0]+ [i % self.max_len_token for i in range(self.max_len_token - 1)] + poi_level_pos_id_seq = [0] + [1 + int(i / self.max_len_token) for i in range(self.max_len_token - 1)] + grid_level_pos_id_seq = [0] + [self.num_grid + 1] * (self.max_len_token - 1) + offset = [1] + poi_cate_id_seq = [self.poi_cate_vocab['PAD']] * self.max_len_token + + + else: + cur_len, num_poi = 1, 1 + token_id_seq, attn_mask_seq = [self.cls_token_id], [1] + word_level_pos_id_seq, poi_level_pos_id_seq, grid_level_pos_id_seq = [0], [0], [0] + offset = [cur_len] + poi_cate_id_seq = [self.poi_cate_vocab['PAD']] + + + for poi in poi_list: + poi_name, _, poi_cate_id, poi_x, poi_y = poi + + poi_name = poi_name.lower() + tokenize_output = self.tokenizer(poi_name, add_special_tokens=False) + + token_id = tokenize_output['input_ids'] + attn_mask = tokenize_output['attention_mask'] + + token_id.append(self.sep_token_id) # add sep token behind each poi + attn_mask.append(1) + + word_level_pos_id = list(range(len(token_id))) + + poi_level_pos_id = [num_poi] * len(token_id) + + grid_x = int(self.num_grid_x * (poi_x - region_coord[0]) / (region_coord[2] - region_coord[0])) + grid_y = int(self.num_grid_y * (region_coord[1] - poi_y) / (region_coord[1] - region_coord[3])) + grid_x, grid_y = min(grid_x, self.num_grid_x - 1), min(grid_y, self.num_grid_y - 1) + grid_id = self.num_grid_x * grid_y + grid_x + 1 + grid_level_pos_id = [grid_id] * len(token_id) + + poi_cate_id = [poi_cate_id] * len(token_id) + + cur_len += len(token_id) + if cur_len <= self.max_len_token: + offset.append(cur_len) + token_id_seq.extend(token_id) + attn_mask_seq.extend(attn_mask) + word_level_pos_id_seq.extend(word_level_pos_id) + poi_level_pos_id_seq.extend(poi_level_pos_id) + grid_level_pos_id_seq.extend(grid_level_pos_id) + poi_cate_id_seq.extend(poi_cate_id) + num_poi += 1 + + else: + break + + ## padding + padding_len = self.max_len_token - offset[-1] + if padding_len > 0: + token_id_seq.extend([self.pad_token_id] * padding_len) + attn_mask_seq.extend([0] * padding_len) + word_level_pos_id_seq.extend([i % self.max_len_token for i in range(padding_len)]) + poi_level_pos_id_seq.extend([num_poi + int(i / self.max_len_poi) for i in range(padding_len)]) + grid_level_pos_id_seq.extend([self.num_grid + 1] * padding_len) + poi_cate_id_seq.extend([self.poi_cate_vocab['PAD']] * padding_len) + + return token_id_seq, attn_mask_seq, word_level_pos_id_seq, poi_level_pos_id_seq, grid_level_pos_id_seq, poi_cate_id_seq + + + + def __getitem__(self, index): + id = self.id_of_region[index] + label = self.label_list[index] + + poi_name_token_ids, attn_mask_poi, word_level_pos_ids, poi_level_pos_ids, grid_level_pos_ids, poi_cate_ids = self.tokenize_poi_data(index) + + img = pil_loader(self.img_path + '{id}.png'.format(id=id)) + img = np.array(img).astype('float32') / 255.0 + img = img.transpose(2, 0, 1) + img = self.img_trans(img) + + poi_name_token_ids = np.array(poi_name_token_ids) + attn_mask_poi = np.array(attn_mask_poi) + word_level_pos_ids = np.array(word_level_pos_ids) + poi_level_pos_ids = np.array(poi_level_pos_ids) + grid_level_pos_ids = np.array(grid_level_pos_ids) + poi_cate_ids = np.array(poi_cate_ids) + + return label, poi_name_token_ids, attn_mask_poi, word_level_pos_ids, poi_level_pos_ids, grid_level_pos_ids, poi_cate_ids, img + + + def __len__(self): + return len(self.id_of_region) + diff --git a/research/ReFound/code/models.py b/research/ReFound/code/models.py new file mode 100644 index 0000000..e2cb68e --- /dev/null +++ b/research/ReFound/code/models.py @@ -0,0 +1,588 @@ +import paddle +from paddle import nn +import paddle.nn.functional as F +from functools import partial +from utils import * +from MoGETransformer import * +from collections import Counter + + + +class PreTrainedModel(nn.Layer): + def __init__(self, config): + super().__init__() + self.config = config + + def _init_weights(self, module): + normal_init = nn.initializer.Normal(mean=0.0, std=self.config['initializer_range']) + zero_init = nn.initializer.Constant(0.) + one_init = nn.initializer.Constant(1.) + + if isinstance(module, nn.Linear): + normal_init(module.weight) + if module.bias is not None: + zero_init(module.bias) + elif isinstance(module, nn.Embedding): + normal_init(module.weight) + if module._padding_idx is not None: + with paddle.no_grad(): + module.weight[module._padding_idx] = 0 + elif isinstance(module, nn.LayerNorm): + zero_init(module.bias) + one_init(module.weight) + + + +class POIEmbed(nn.Layer): + def __init__(self, config): + super().__init__() + self.config = config + self.seq_len_poi = config['max_len_token'] + self.num_grid = config['num_grid_x'] * config['num_grid_y'] + + self.word_embedding_table = nn.Embedding(config['vocab_size'], config['hidden_size'], padding_idx=0) # from bert + self.token_type_embedding_table = nn.Embedding(config['type_vocab_size'], config['hidden_size']) # from bert + self.word_level_pos_embedding_table = nn.Embedding(config['max_len_token'], config['hidden_size']) + self.poi_level_pos_embedding_table = nn.Embedding(config['max_len_poi'], config['hidden_size']) + self.grid_level_pos_embedding_table = nn.Embedding(self.num_grid + 2, config['hidden_size']) + self.poi_cate_embedding_table = nn.Embedding(config['poi_cate_num'], config['hidden_size']) + + self.LayerNorm = nn.LayerNorm(config['hidden_size'], epsilon=config['layer_norm_eps']) + self.dropout = nn.Dropout(config['hidden_dropout_prob']) + + def forward(self, poi_name_token_ids, word_level_pos_ids, poi_level_pos_ids, grid_level_pos_ids, poi_cate_ids): + poi_name_token_embedding = self.word_embedding_table(poi_name_token_ids) + token_type_embedding = self.token_type_embedding_table(paddle.zeros_like(word_level_pos_ids)) + word_level_pos_embedding = self.word_level_pos_embedding_table(word_level_pos_ids) + poi_level_pos_embedding = self.poi_level_pos_embedding_table(poi_level_pos_ids) + grid_level_pos_embedding = self.grid_level_pos_embedding_table(grid_level_pos_ids) + poi_cate_embedding = self.poi_cate_embedding_table(poi_cate_ids) + + poi_embedding = \ + poi_name_token_embedding + \ + token_type_embedding + \ + word_level_pos_embedding + \ + poi_level_pos_embedding + \ + grid_level_pos_embedding + \ + poi_cate_embedding + + poi_embedding = self.LayerNorm(poi_embedding) + poi_embedding = self.dropout(poi_embedding) + return poi_embedding + + + + +class SateEmbed(nn.Layer): + def __init__(self, config): + super().__init__() + self.config = config + self.num_channel = 3 + image_height, image_width = pair(config['image_size']) + patch_height, patch_width = pair(config['patch_size']) + assert image_height % patch_height == 0 and image_width % patch_width == 0, \ + 'Image dimensions must be divisible by the patch size.' + self.num_patch = (image_height // patch_height) * (image_width // patch_width) + self.seq_len_img = self.num_patch + 1 + self.num_patch_pixel = patch_height * patch_width + self.patch_dim = 3 * patch_height * patch_width + + self.img2patch_unfold = nn.Unfold( + kernel_sizes=[patch_height, patch_width], + strides=[patch_height, patch_width], + paddings=0, + dilations=1 + ) + + self.patch_to_emb = nn.Linear(self.patch_dim, config['hidden_size']) + self.pos_1d_embedding_table = nn.Embedding(self.seq_len_img, config['hidden_size']) + + self.cls_token = self.create_parameter( + shape=[1, 1, config['hidden_size']], is_bias=False, + default_initializer=nn.initializer.Normal(mean=0.0, std=self.config['initializer_range']) + ) + self.mask_token = self.create_parameter( + shape=[config['hidden_size']], is_bias=False, + default_initializer=nn.initializer.Normal(mean=0.0, std=self.config['initializer_range']) + ) + + self.LayerNorm = nn.LayerNorm(config['hidden_size'], epsilon=config['layer_norm_eps']) + self.dropout = nn.Dropout(config['hidden_dropout_prob']) + + + def img2patch_func(self, img): + batch_size, num_channel = img.shape[:2] + patch = self.img2patch_unfold(img) + patch = patch.reshape((batch_size, num_channel, self.num_patch_pixel, self.num_patch)) + patch = patch.transpose((0, 3, 2, 1)).reshape((batch_size, self.num_patch, self.patch_dim)) + return patch + + + def forward(self, img, mask, masking): + patch = self.img2patch_func(img) + patch_embedding = self.patch_to_emb(patch) + + if masking: + assert mask is not None + patch_embedding[mask] = self.mask_token + + batch_size = patch_embedding.shape[0] + cls_tokens = self.cls_token.expand((batch_size, -1, -1)) + patch_embedding = paddle.concat([cls_tokens, patch_embedding], axis=1) + + pos_1d = paddle.arange(0, patch_embedding.shape[1]).unsqueeze(0).expand((batch_size, -1)) + pos_embedding_1d = self.pos_1d_embedding_table(pos_1d) + patch_embedding = patch_embedding + pos_embedding_1d + + patch_embedding = self.LayerNorm(patch_embedding) + patch_embedding = self.dropout(patch_embedding) + + return patch_embedding + + + + +class ModalEmbed(nn.Layer): + def __init__(self, config): + super().__init__() + self.config = config + self.mod_embed_tabel = nn.Embedding(2, config['hidden_size']) + + def forward(self, poi_embedding, img_embedding): + batch_size = poi_embedding.shape[0] + seq_len_poi = poi_embedding.shape[1] + seq_len_img = img_embedding.shape[1] + + mod_id_poi = paddle.full([batch_size, seq_len_poi], 0, dtype='int64') + mod_embedding_poi = self.mod_embed_tabel(mod_id_poi) + poi_embedding = poi_embedding + mod_embedding_poi + + mod_id_img = paddle.full([batch_size, seq_len_img], 1, dtype='int64') + mod_embedding_img = self.mod_embed_tabel(mod_id_img) + img_embedding = img_embedding + mod_embedding_img + + return poi_embedding, img_embedding + + + + +class ReFound(PreTrainedModel): + def __init__(self, config): + super().__init__(config) + self.config = config + + self.poi_embed_module = POIEmbed(config) + self.img_embed_module = SateEmbed(config) + self.mod_embed_module = ModalEmbed(config) + self.transformer = MoGEEncoder(config) + + + def prepare_poi_data(self, poi_data, masking_poi): + + if masking_poi: + poi_name_token_ids = poi_data['poi_name_token_ids_masked'] + else: + poi_name_token_ids = poi_data['poi_name_token_ids'] + + attn_mask_poi = poi_data['attn_mask_poi'] + word_level_pos_ids = poi_data['word_level_pos_ids'] + poi_level_pos_ids = poi_data['poi_level_pos_ids'] + grid_level_pos_ids = poi_data['grid_level_pos_ids'] + poi_cate_ids = poi_data['poi_cate_ids'] + + + prepared_poi_data = { + 'poi_name_token_ids': poi_name_token_ids, + 'attn_mask_poi': attn_mask_poi, + 'word_level_pos_ids': word_level_pos_ids, + 'poi_level_pos_ids': poi_level_pos_ids, + 'grid_level_pos_ids': grid_level_pos_ids, + 'poi_cate_ids': poi_cate_ids, + } + return prepared_poi_data + + + def prepare_img_data(self, img_data, masking_img): + + seq_len_img = self.img_embed_module.seq_len_img + + if masking_img: + mask = img_data['img_mask'] + else: + mask = None + + img = img_data['img'] + batch_size = img.shape[0] + attn_mask_img = paddle.ones((batch_size, seq_len_img), dtype='int64') + + prepared_img_data = { + 'img': img, + 'mask': mask, + 'attn_mask_img': attn_mask_img, + } + return prepared_img_data + + + def forward( + self, + poi_data, + img_data, + masking_poi, + masking_img + ): + + prepared_poi_data = self.prepare_poi_data( + poi_data=poi_data, + masking_poi=masking_poi, + ) + + poi_embedding = self.poi_embed_module( + poi_name_token_ids=prepared_poi_data['poi_name_token_ids'], + word_level_pos_ids=prepared_poi_data['word_level_pos_ids'], + poi_level_pos_ids=prepared_poi_data['poi_level_pos_ids'], + grid_level_pos_ids=prepared_poi_data['grid_level_pos_ids'], + poi_cate_ids=prepared_poi_data['poi_cate_ids'], + ) + attn_mask_poi = prepared_poi_data['attn_mask_poi'] + + + prepared_img_data = self.prepare_img_data( + img_data=img_data, + masking_img=masking_img, + ) + + img_embedding = self.img_embed_module( + img=prepared_img_data['img'], + mask=prepared_img_data['mask'], + masking=masking_img, + ) + attn_mask_img = prepared_img_data['attn_mask_img'] + + + poi_embedding, img_embedding = self.mod_embed_module(poi_embedding, img_embedding) + all_embedding = paddle.concat([poi_embedding, img_embedding], axis=1) + attn_mask = paddle.concat([attn_mask_poi, attn_mask_img], axis=1) + split_idx = poi_embedding.shape[1] + assert split_idx == self.poi_embed_module.seq_len_poi + + extended_attn_mask = get_extended_attention_mask(attn_mask) + + encoder_output = self.transformer( + hidden_states=all_embedding, + attention_mask=extended_attn_mask, + split_idx=split_idx, + ) + + return encoder_output + + + +class AttnPool(nn.Layer): + def __init__(self, config, hidden_size=32): + super(AttnPool, self).__init__() + + self.l1 = nn.Linear(config['hidden_size'], hidden_size) + self.ac = nn.Tanh() + self.l2 = nn.Linear(int(hidden_size), 1, bias_attr=False) + + def forward(self, z): + w = self.l1(z) + w = self.ac(w) + w = self.l2(w) + beta = F.softmax(w, axis=1) + return (beta * z).sum(1) + + + +class UrbanVillageDetectionHead(nn.Layer): + def __init__(self, config): + super().__init__() + self.config = config + self.dense = nn.Linear(config['hidden_size'], config['hidden_size']) + self.act_fn = nn.GELU() + self.dropout = nn.Dropout(config['fdrop']) + self.decoder = nn.Linear(config['hidden_size'], 1) + self.sigmoid = nn.Sigmoid() + + def forward(self, hidden_states): + output = self.dense(hidden_states) + output = self.act_fn(output) + output = self.dropout(output) + output = self.decoder(output) + output = self.sigmoid(output) + return output + + + +class FinetuneUrbanVillageDetection(PreTrainedModel): + def __init__(self, config): + super().__init__(config) + + self.encoder = ReFound(config) + self.target_prediction = UrbanVillageDetectionHead(config) + + if config['agg'] == 'attn': + self.attn_agg = AttnPool(config) + + self.apply(self._init_weights) + + def forward(self, poi_data, img_data): + + encoder_output = self.encoder( + poi_data=poi_data, + img_data=img_data, + masking_poi=False, + masking_img=False, + ) + + if self.config['agg'] == 'avr': + seq_len_img = self.encoder.img_embed_module.seq_len_img + cls_output_poi = encoder_output[:, 0] + cls_output_img = encoder_output[:, -seq_len_img] + agg_output = 0.5 * (cls_output_poi + cls_output_img) + + elif self.config['agg'] == 'attn': + seq_len_img = self.encoder.img_embed_module.seq_len_img + cls_output_poi = encoder_output[:, 0] + cls_output_img = encoder_output[:, -seq_len_img] + agg_output = paddle.stack([cls_output_poi, cls_output_img], axis=1) + agg_output = self.attn_agg(agg_output) + + prediction_scores = self.target_prediction(agg_output) + return prediction_scores + + + +class CommercialActivenessPredictionHead(nn.Layer): + def __init__(self, config): + super().__init__() + self.config = config + self.dense = nn.Linear(config['hidden_size'], config['hidden_size']) + self.act_fn = nn.GELU() + self.dropout = nn.Dropout(config['fdrop']) + self.decoder = nn.Linear(config['hidden_size'], 1) + + def forward(self, hidden_states): + output = self.dense(hidden_states) + output = self.act_fn(output) + output = self.dropout(output) + output = self.decoder(output) + return output + + + +class FinetuneCommercialActivenessPrediction(PreTrainedModel): + def __init__(self, config): + super().__init__(config) + + self.encoder = ReFound(config) + self.target_prediction = CommercialActivenessPredictionHead(config) + + if config['agg'] == 'attn': + self.attn_agg = AttnPool(config) + + self.apply(self._init_weights) + + def forward(self, poi_data, img_data): + + encoder_output = self.encoder( + poi_data=poi_data, + img_data=img_data, + masking_poi=False, + masking_img=False, + ) + + if self.config['agg'] == 'avr': + seq_len_img = self.encoder.img_embed_module.seq_len_img + cls_output_poi = encoder_output[:, 0] + cls_output_img = encoder_output[:, -seq_len_img] + agg_output = 0.5 * (cls_output_poi + cls_output_img) + + elif self.config['agg'] == 'attn': + seq_len_img = self.encoder.img_embed_module.seq_len_img + cls_output_poi = encoder_output[:, 0] + cls_output_img = encoder_output[:, -seq_len_img] + agg_output = paddle.stack([cls_output_poi, cls_output_img], axis=1) + agg_output = self.attn_agg(agg_output) + + prediction_scores = self.target_prediction(agg_output) + return prediction_scores + + + +class PopulationPredictionHead(nn.Layer): + def __init__(self, config): + super().__init__() + self.config = config + self.dense = nn.Linear(config['hidden_size'], config['hidden_size']) + self.act_fn = nn.GELU() + self.dropout = nn.Dropout(config['fdrop']) + self.decoder = nn.Linear(config['hidden_size'], 1) + + def forward(self, hidden_states): + output = self.dense(hidden_states) + output = self.act_fn(output) + output = self.dropout(output) + output = self.decoder(output) + return output + + + +class FinetunePopulationPrediction(PreTrainedModel): + def __init__(self, config): + super().__init__(config) + + self.encoder = ReFound(config) + self.target_prediction = PopulationPredictionHead(config) + + if config['agg'] == 'attn': + self.attn_agg = AttnPool(config) + + self.apply(self._init_weights) + + def forward(self, poi_data, img_data): + + encoder_output = self.encoder( + poi_data=poi_data, + img_data=img_data, + masking_poi=False, + masking_img=False, + ) + + if self.config['agg'] == 'avr': + seq_len_img = self.encoder.img_embed_module.seq_len_img + cls_output_poi = encoder_output[:, 0] + cls_output_img = encoder_output[:, -seq_len_img] + agg_output = 0.5 * (cls_output_poi + cls_output_img) + + elif self.config['agg'] == 'attn': + seq_len_img = self.encoder.img_embed_module.seq_len_img + cls_output_poi = encoder_output[:, 0] + cls_output_img = encoder_output[:, -seq_len_img] + agg_output = paddle.stack([cls_output_poi, cls_output_img], axis=1) + agg_output = self.attn_agg(agg_output) + + prediction_scores = self.target_prediction(agg_output) + return prediction_scores + + + +class FeatureExtractor(PreTrainedModel): + # extract region representation for feature-based prediction + def __init__(self, config): + super().__init__(config) + + self.encoder = ReFound(config) + self.apply(self._init_weights) + + def forward(self, poi_data, img_data): + + encoder_output = self.encoder( + poi_data=poi_data, + img_data=img_data, + masking_poi=False, + masking_img=False, + ) + + seq_len_img = self.encoder.img_embed_module.seq_len_img + cls_output_poi = encoder_output[:, 0] + cls_output_img = encoder_output[:, -seq_len_img] + cls_output = paddle.stack([cls_output_poi, cls_output_img], axis=1) + + return cls_output + + + +class FeatureBasedUrbanVillageDetection(PreTrainedModel): + def __init__(self, config): + super().__init__(config) + + self.target_prediction = UrbanVillageDetectionHead(config) + + if config['agg'] == 'attn': + self.attn_agg = AttnPool(config) + + self.apply(self._init_weights) + + def forward(self, region_emb): + + if self.config['agg'] == 'avr': + cls_output_poi = region_emb[:, 0] + cls_output_img = region_emb[:, 1] + agg_output = 0.5 * (cls_output_poi + cls_output_img) + + elif self.config['agg'] == 'attn': + cls_output_poi = region_emb[:, 0] + cls_output_img = region_emb[:, 1] + agg_output = paddle.stack([cls_output_poi, cls_output_img], axis=1) + agg_output = self.attn_agg(agg_output) + + prediction_scores = self.target_prediction(agg_output) + + return prediction_scores + + + + +class FeatureBasedCommercialActivenessPrediction(PreTrainedModel): + def __init__(self, config): + super().__init__(config) + + self.target_prediction = CommercialActivenessPredictionHead(config) + + if config['agg'] == 'attn': + self.attn_agg = AttnPool(config) + + self.apply(self._init_weights) + + def forward(self, region_emb): + + if self.config['agg'] == 'avr': + cls_output_poi = region_emb[:, 0] + cls_output_img = region_emb[:, 1] + agg_output = 0.5 * (cls_output_poi + cls_output_img) + + elif self.config['agg'] == 'attn': + cls_output_poi = region_emb[:, 0] + cls_output_img = region_emb[:, 1] + agg_output = paddle.stack([cls_output_poi, cls_output_img], axis=1) + agg_output = self.attn_agg(agg_output) + + prediction_scores = self.target_prediction(agg_output) + + return prediction_scores + + + + + +class FeatureBasedPopulationPrediction(PreTrainedModel): + def __init__(self, config): + super().__init__(config) + + self.target_prediction = PopulationPredictionHead(config) + + if config['agg'] == 'attn': + self.attn_agg = AttnPool(config) + + self.apply(self._init_weights) + + def forward(self, region_emb): + + if self.config['agg'] == 'avr': + cls_output_poi = region_emb[:, 0] + cls_output_img = region_emb[:, 1] + agg_output = 0.5 * (cls_output_poi + cls_output_img) + + elif self.config['agg'] == 'attn': + cls_output_poi = region_emb[:, 0] + cls_output_img = region_emb[:, 1] + agg_output = paddle.stack([cls_output_poi, cls_output_img], axis=1) + agg_output = self.attn_agg(agg_output) + + prediction_scores = self.target_prediction(agg_output) + + return prediction_scores + + + diff --git a/research/ReFound/code/script_feature_based.sh b/research/ReFound/code/script_feature_based.sh new file mode 100644 index 0000000..83cbaf9 --- /dev/null +++ b/research/ReFound/code/script_feature_based.sh @@ -0,0 +1,67 @@ +#!/bin/bash + +shparam1=${1} +shparam2=${2} +shparam3=${3} +shparam4=${4} +shparam5=${5} +shparam6=${6} +shparam7=${7} +shparam8=${8} + + +function train_uvd() { + CUDA_VISIBLE_DEVICES=${7} \ + python -u feature_based_uvd.py \ + --city ${1} \ + --checkpoint '299' \ + --agg ${2} \ + --fdrop ${3} \ + --fbatch_size '12' \ + --epoch_num '40' \ + --warmup_epochs '3' \ + --flr ${4} \ + --fdecay ${5} \ + --min_lr '0' \ + --accum_iter '1' \ + --seed ${6} +} + + +function train_cap() { + CUDA_VISIBLE_DEVICES=${7} \ + python -u feature_based_cap.py \ + --city ${1} \ + --checkpoint '299' \ + --agg ${2} \ + --fdrop ${3} \ + --fbatch_size '12' \ + --epoch_num '40' \ + --warmup_epochs '3' \ + --flr ${4} \ + --fdecay ${5} \ + --min_lr '0' \ + --accum_iter '1' \ + --seed ${6} +} + + +function train_pop() { + CUDA_VISIBLE_DEVICES=${7} \ + python -u feature_based_pop.py \ + --city ${1} \ + --checkpoint '299' \ + --agg ${2} \ + --fdrop ${3} \ + --fbatch_size '12' \ + --epoch_num '40' \ + --warmup_epochs '3' \ + --flr ${4} \ + --fdecay ${5} \ + --min_lr '0' \ + --accum_iter '1' \ + --seed ${6} +} + + +train_${shparam1} ${shparam2} ${shparam3} ${shparam4} ${shparam5} ${shparam6} ${shparam7} ${shparam8} diff --git a/research/ReFound/code/script_finetune.sh b/research/ReFound/code/script_finetune.sh new file mode 100644 index 0000000..e91827d --- /dev/null +++ b/research/ReFound/code/script_finetune.sh @@ -0,0 +1,67 @@ +#!/bin/bash + +shparam1=${1} +shparam2=${2} +shparam3=${3} +shparam4=${4} +shparam5=${5} +shparam6=${6} +shparam7=${7} +shparam8=${8} + + +function train_uvd() { + CUDA_VISIBLE_DEVICES=${7} \ + python -u finetune_uvd.py \ + --city ${1} \ + --checkpoint '299' \ + --agg ${2} \ + --fdrop ${3} \ + --fbatch_size '12' \ + --epoch_num '30' \ + --warmup_epochs '3' \ + --flr ${4} \ + --fdecay ${5} \ + --min_lr '0' \ + --accum_iter '1' \ + --seed ${6} +} + + +function train_cap() { + CUDA_VISIBLE_DEVICES=${7} \ + python -u finetune_cap.py \ + --city ${1} \ + --checkpoint '299' \ + --agg ${2} \ + --fdrop ${3} \ + --fbatch_size '12' \ + --epoch_num '40' \ + --warmup_epochs '3' \ + --flr ${4} \ + --fdecay ${5} \ + --min_lr '0' \ + --accum_iter '1' \ + --seed ${6} +} + + +function train_pop() { + CUDA_VISIBLE_DEVICES=${7} \ + python -u finetune_pop.py \ + --city ${1} \ + --checkpoint '299' \ + --agg ${2} \ + --fdrop ${3} \ + --fbatch_size '12' \ + --epoch_num '40' \ + --warmup_epochs '3' \ + --flr ${4} \ + --fdecay '0.01' \ + --min_lr '0' \ + --accum_iter '1' \ + --seed ${6} +} + + +train_${shparam1} ${shparam2} ${shparam3} ${shparam4} ${shparam5} ${shparam6} ${shparam7} ${shparam8} diff --git a/research/ReFound/code/utils.py b/research/ReFound/code/utils.py new file mode 100644 index 0000000..08f7291 --- /dev/null +++ b/research/ReFound/code/utils.py @@ -0,0 +1,110 @@ +# -*- coding: utf-8 -*- + +import numpy as np +import random +import math +import json +import PIL.Image as pil +import paddle + + +def pil_loader(path): + with open(path, "rb") as img_f: + img = pil.open(img_f) + return img.convert("RGB") + + +def seed_setup(seed): + # seed = seed + utils_dist.get_rank() + np.random.seed(seed) + random.seed(seed) + paddle.seed(seed) + + +def pair(t): + return t if isinstance(t, tuple) else (t, t) + + +def get_extended_attention_mask(attention_mask): + dtype = 'float32' + min_value = -3.4028234663852886e+38 + extended_attention_mask = paddle.unsqueeze(attention_mask, axis=[1, 2]) + extended_attention_mask = paddle.cast(extended_attention_mask, dtype=dtype) + extended_attention_mask = (1.0 - extended_attention_mask) * min_value + return extended_attention_mask + + +def adjust_learning_rate(optimizer, epoch, warmup_epochs, epoch_num, peak_lr, min_lr): + + if epoch < warmup_epochs: + lr = peak_lr * epoch / warmup_epochs + else: + lr = min_lr + (peak_lr - min_lr) * (epoch_num - epoch) / (epoch_num - warmup_epochs) + + optimizer.set_lr(lr) + + +def param_groups_lrd(model, weight_decay=0.05, no_weight_decay_list=[], layer_decay=.75): + """ + Parameter groups for layer-wise lr decay + Following BEiT: https://github.com/microsoft/unilm/blob/master/beit/optim_factory.py#L58 + """ + param_group_names = {} + param_groups = {} + + num_layers = len(model.encoder.transformer.layer) + 1 + + layer_scales = list(layer_decay ** (num_layers - i) for i in range(num_layers + 1)) + + for n, p in model.named_parameters(): + if p.stop_gradient: + continue + + if p.ndim == 1 or n in no_weight_decay_list: + g_decay = "no_decay" + this_decay = 0. + else: + g_decay = "decay" + this_decay = weight_decay + + layer_id = get_layer_id(n, num_layers) + group_name = "layer_%d_%s" % (layer_id, g_decay) + + if group_name not in param_group_names: + this_scale = layer_scales[layer_id] + + param_group_names[group_name] = { + "learning_rate": this_scale, + "weight_decay": this_decay, + "params": [], + } + param_groups[group_name] = { + "learning_rate": this_scale, + "weight_decay": this_decay, + "params": [], + } + + param_group_names[group_name]["params"].append(n) + param_groups[group_name]["params"].append(p) + + return list(param_groups.values()) + + +def get_layer_id(name, num_layers): + if name.startswith('encoder'): + if name.startswith('encoder.poi_embed_module'): + return 0 + elif name.startswith('encoder.img_embed_module'): + return 0 + elif name.startswith('encoder.mod_embed_module'): + return 0 + elif name.startswith('encoder.transformer'): + return int(name.split('.')[3]) + 1 + + elif name.startswith('target_prediction'): + return num_layers + + elif name.startswith('attn_agg'): + return num_layers + + diff --git a/research/ReFound/requirements.txt b/research/ReFound/requirements.txt new file mode 100644 index 0000000..2ed9684 --- /dev/null +++ b/research/ReFound/requirements.txt @@ -0,0 +1,5 @@ +numpy +PIL +paddlepaddle-gpu==2.4.2 +scikit-learn +scipy \ No newline at end of file