Skip to content

Commit bdfd24d

Browse files
ThisSeanZhangThisSeanZhang
committed
add cache for route forwarding
1 parent 1ad5db4 commit bdfd24d

File tree

12 files changed

+373
-37
lines changed

12 files changed

+373
-37
lines changed

‎landscape-ebpf/src/bin/land.rs‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,4 @@
1+
// cargo run --package landscape-ebpf --bin land
12
pub fn main() {
23
landscape_ebpf::landscape::test();
34
}

‎landscape-ebpf/src/bpf/firewall.bpf.c‎

Lines changed: 11 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -453,7 +453,9 @@ static int timer_clean_callback(void *map_mapping_timer_, struct firewall_conntr
453453
// bpf_log_info("call back remove conn");
454454
ret = bpf_map_delete_elem(&fire2_conn_map, key);
455455
if (ret) {
456-
bpf_log_error("call back remove conn error: %pI4:%d->%pI4:%d", &key->local_addr, bpf_ntohs(key->local_port), &value->trigger_addr, bpf_ntohs(value->trigger_port));
456+
bpf_log_error("call back remove conn error: %pI4:%d->%pI4:%d", &key->local_addr,
457+
bpf_ntohs(key->local_port), &value->trigger_addr,
458+
bpf_ntohs(value->trigger_port));
457459
}
458460
return 0;
459461
}
@@ -1061,6 +1063,10 @@ int ipv4_ingress_firewall(struct __sk_buff *skb) {
10611063
// "packet ip:%pI4:%d->%pI4:%d, ip_protocol: %d", &packet_info.ip_hdr.pair_ip.src_addr,
10621064
// bpf_ntohs(packet_info.ip_hdr.pair_ip.src_port), &packet_info.ip_hdr.pair_ip.dst_addr,
10631065
// bpf_ntohs(packet_info.ip_hdr.pair_ip.dst_port), packet_info.ip_hdr.ip_protocol);
1066+
u32 mark = skb->mark;
1067+
barrier_var(mark);
1068+
skb->mark = replace_cache_mask(mark, INGRESS_STATIC_MARK);
1069+
// bpf_log_info("set wan ingress mark: %u", skb->mark);
10641070
return TC_ACT_UNSPEC;
10651071
}
10661072
return TC_ACT_SHOT;
@@ -1241,6 +1247,10 @@ int ipv6_ingress_firewall(struct __sk_buff *skb) {
12411247
// "packet ip:%pI4:%d->%pI4:%d, ip_protocol: %d", &packet_info.ip_hdr.pair_ip.src_addr,
12421248
// bpf_ntohs(packet_info.ip_hdr.pair_ip.src_port), &packet_info.ip_hdr.pair_ip.dst_addr,
12431249
// bpf_ntohs(packet_info.ip_hdr.pair_ip.dst_port), packet_info.ip_hdr.ip_protocol);
1250+
u32 mark = skb->mark;
1251+
barrier_var(mark);
1252+
skb->mark = replace_cache_mask(mark, INGRESS_STATIC_MARK);
1253+
// bpf_log_info("set wan ingress mark: %u", skb->mark);
12441254
return TC_ACT_UNSPEC;
12451255
}
12461256
return TC_ACT_SHOT;

‎landscape-ebpf/src/bpf/flow.h‎

Lines changed: 119 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -50,7 +50,6 @@ struct {
5050
__uint(max_entries, 2048);
5151
} flow_target_map SEC(".maps");
5252

53-
5453
// struct each_flow_target {
5554
// __uint(type, BPF_MAP_TYPE_HASH);
5655
// __uint(map_flags, BPF_F_NO_PREALLOC);
@@ -202,22 +201,126 @@ struct {
202201
__uint(pinning, LIBBPF_PIN_BY_NAME);
203202
} rt_target_map SEC(".maps");
204203

205-
// struct lan_mac_cache_key {
206-
// u8 ip[16];
207-
// u8 l3_protocol;
208-
// u8 _pad[3];
209-
// };
204+
struct rt_cache_key {
205+
struct in6_addr local_addr;
206+
struct in6_addr remote_addr;
207+
} __rt_cache_key;
210208

211-
// struct lan_mac_cache {
212-
// u8 mac[6];
213-
// u8 _pad[2];
214-
// };
209+
struct rt_cache_value {
210+
union {
211+
__u32 mark_value;
212+
__u32 ifindex;
213+
};
214+
} __rt_cache_value;
215215

216-
// struct {
217-
// __uint(type, BPF_MAP_TYPE_LRU_HASH);
218-
// __type(key, struct lan_mac_cache_key);
219-
// __type(value, struct lan_mac_cache);
220-
// __uint(max_entries, 65535);
221-
// } ip_mac_tab SEC(".maps");
216+
#define WAN_CACHE 0
217+
#define LAN_CACHE 1
218+
219+
// 缓存
220+
struct each_cache_hash {
221+
__uint(type, BPF_MAP_TYPE_LRU_HASH);
222+
__type(key, struct rt_cache_key);
223+
__type(value, struct rt_cache_value);
224+
__uint(max_entries, 65536);
225+
} __each_cache_map SEC(".maps");
226+
227+
// flow <-> 对应规则 map
228+
struct {
229+
__uint(type, BPF_MAP_TYPE_ARRAY_OF_MAPS);
230+
__type(key, u32);
231+
__uint(max_entries, 4);
232+
__uint(pinning, LIBBPF_PIN_BY_NAME);
233+
__array(values, struct each_cache_hash);
234+
} rt_cache_map SEC(".maps");
235+
236+
static __always_inline int setting_cache_in_wan(const struct route_context *context, u32 ifindex) {
237+
#define BPF_LOG_TOPIC "setting_cache_in_wan"
238+
struct rt_cache_key search_key = {0};
239+
u32 key = LAN_CACHE;
240+
COPY_ADDR_FROM(search_key.local_addr.in6_u.u6_addr8, context->daddr.in6_u.u6_addr8);
241+
COPY_ADDR_FROM(search_key.remote_addr.in6_u.u6_addr8, context->saddr.in6_u.u6_addr8);
242+
243+
void *lan_cache = bpf_map_lookup_elem(&rt_cache_map, &key);
244+
if (lan_cache) {
245+
struct rt_cache_value *target = bpf_map_lookup_elem(lan_cache, &search_key);
246+
if (target) {
247+
// if (context->l3_protocol == LANDSCAPE_IPV4_TYPE) {
248+
// bpf_log_info("Already cached %pI4 -> %pI4", search_key.local_addr.in6_u.u6_addr8,
249+
// search_key.remote_addr.in6_u.u6_addr8);
250+
// } else {
251+
// bpf_log_info("Already cached %pI6 -> %pI6", search_key.local_addr.in6_u.u6_addr8,
252+
// search_key.remote_addr.in6_u.u6_addr8);
253+
// }
254+
return TC_ACT_OK;
255+
}
256+
}
257+
258+
key = WAN_CACHE;
259+
void *wan_cache = bpf_map_lookup_elem(&rt_cache_map, &key);
260+
if (wan_cache) {
261+
struct rt_cache_value *target = bpf_map_lookup_elem(wan_cache, &search_key);
262+
if (target) {
263+
target->ifindex = ifindex;
264+
} else {
265+
struct rt_cache_value new_target_cache = {0};
266+
new_target_cache.ifindex = ifindex;
267+
bpf_map_update_elem(wan_cache, &search_key, &new_target_cache, BPF_ANY);
268+
}
269+
270+
// if (context->l3_protocol == LANDSCAPE_IPV4_TYPE) {
271+
// bpf_log_info("cache %pI4 -> %pI4", search_key.local_addr.in6_u.u6_addr8,
272+
// search_key.remote_addr.in6_u.u6_addr8);
273+
// } else {
274+
// bpf_log_info("cache %pI6 -> %pI6", search_key.local_addr.in6_u.u6_addr8,
275+
// search_key.remote_addr.in6_u.u6_addr8);
276+
// }
277+
} else {
278+
bpf_log_info("could not find wan_cache: %d", key);
279+
}
280+
281+
return TC_ACT_OK;
282+
#undef BPF_LOG_TOPIC
283+
}
284+
285+
static __always_inline int setting_cache_in_lan(const struct route_context *context,
286+
u32 flow_mark) {
287+
#define BPF_LOG_TOPIC "setting_cache_in_lan"
288+
struct rt_cache_key search_key = {0};
289+
u32 key = WAN_CACHE;
290+
COPY_ADDR_FROM(search_key.local_addr.in6_u.u6_addr8, context->saddr.in6_u.u6_addr8);
291+
COPY_ADDR_FROM(search_key.remote_addr.in6_u.u6_addr8, context->daddr.in6_u.u6_addr8);
292+
293+
void *wan_cache = bpf_map_lookup_elem(&rt_cache_map, &key);
294+
if (wan_cache) {
295+
struct rt_cache_value *target = bpf_map_lookup_elem(wan_cache, &search_key);
296+
if (target) {
297+
return TC_ACT_OK;
298+
}
299+
}
300+
301+
key = LAN_CACHE;
302+
void *lan_cache = bpf_map_lookup_elem(&rt_cache_map, &key);
303+
if (lan_cache) {
304+
struct rt_cache_value *target = bpf_map_lookup_elem(lan_cache, &search_key);
305+
if (target) {
306+
target->mark_value = flow_mark;
307+
} else {
308+
struct rt_cache_value new_target_cache = {0};
309+
new_target_cache.mark_value = flow_mark;
310+
bpf_map_update_elem(lan_cache, &search_key, &new_target_cache, BPF_ANY);
311+
}
312+
313+
// if (context->l3_protocol == LANDSCAPE_IPV4_TYPE) {
314+
// bpf_log_info("cache %pI4 -> %pI4", search_key.local_addr.in6_u.u6_addr8,
315+
// search_key.remote_addr.in6_u.u6_addr8);
316+
// } else {
317+
// bpf_log_info("cache %pI6 -> %pI6", search_key.local_addr.in6_u.u6_addr8,
318+
// search_key.remote_addr.in6_u.u6_addr8);
319+
// }
320+
}
321+
322+
return TC_ACT_OK;
323+
#undef BPF_LOG_TOPIC
324+
}
222325

223326
#endif /* __LD_FLOW_H__ */

‎landscape-ebpf/src/bpf/flow_lan.bpf.c‎

Lines changed: 76 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -14,13 +14,13 @@ char LICENSE[] SEC("license") = "Dual BSD/GPL";
1414
const volatile int current_eth_net_offset = 14;
1515

1616
// todo: 将 flow_target_info 独立出来维护
17-
struct {
18-
__uint(type, BPF_MAP_TYPE_LRU_HASH);
19-
__uint(map_flags, BPF_F_NO_COMMON_LRU);
20-
__type(key, struct old_flow_cache_key);
21-
__type(value, struct flow_target_info);
22-
__uint(max_entries, 4096);
23-
} flow_cache_map SEC(".maps");
17+
// struct {
18+
// __uint(type, BPF_MAP_TYPE_LRU_HASH);
19+
// __uint(map_flags, BPF_F_NO_COMMON_LRU);
20+
// __type(key, struct old_flow_cache_key);
21+
// __type(value, struct flow_target_info);
22+
// __uint(max_entries, 4096);
23+
// } flow_cache_map SEC(".maps");
2424

2525
static __always_inline int is_broadcast_mac(struct __sk_buff *skb) {
2626
u8 mac[6];
@@ -480,6 +480,58 @@ static __always_inline int pick_wan_and_send_by_flow_id(struct __sk_buff *skb,
480480
#undef BPF_LOG_TOPIC
481481
}
482482

483+
static __always_inline int search_route_in_lan(struct __sk_buff *skb,
484+
const int current_eth_net_offset,
485+
const struct route_context *context,
486+
u32 *flow_mark) {
487+
#define BPF_LOG_TOPIC "search_route_in_lan"
488+
int ret = 0;
489+
u32 key = WAN_CACHE;
490+
struct rt_cache_key search_key = {0};
491+
COPY_ADDR_FROM(search_key.local_addr.in6_u.u6_addr8, context->saddr.in6_u.u6_addr8);
492+
COPY_ADDR_FROM(search_key.remote_addr.in6_u.u6_addr8, context->daddr.in6_u.u6_addr8);
493+
494+
// Fist WAN
495+
void *wan_cache = bpf_map_lookup_elem(&rt_cache_map, &key);
496+
if (wan_cache) {
497+
struct rt_cache_value *target = bpf_map_lookup_elem(wan_cache, &search_key);
498+
if (target) {
499+
struct wan_ip_info_key wan_search_key = {0};
500+
wan_search_key.ifindex = target->ifindex;
501+
wan_search_key.l3_protocol = context->l3_protocol;
502+
503+
struct wan_ip_info_value *wan_ip_info =
504+
bpf_map_lookup_elem(&wan_ipv4_binding, &wan_search_key);
505+
if (wan_ip_info != NULL) {
506+
struct bpf_redir_neigh param;
507+
if (context->l3_protocol == LANDSCAPE_IPV4_TYPE) {
508+
param.nh_family = AF_INET;
509+
} else {
510+
param.nh_family = AF_INET6;
511+
}
512+
513+
COPY_ADDR_FROM(param.ipv6_nh, wan_ip_info->gateway.bits);
514+
ret = bpf_redirect_neigh(target->ifindex, &param, sizeof(param), 0);
515+
return ret;
516+
}
517+
}
518+
}
519+
520+
key = LAN_CACHE;
521+
void *lan_cache = bpf_map_lookup_elem(&rt_cache_map, &key);
522+
if (lan_cache) {
523+
struct rt_cache_value *target = bpf_map_lookup_elem(lan_cache, &search_key);
524+
if (target) {
525+
*flow_mark = target->mark_value;
526+
return pick_wan_and_send_by_flow_id(skb, current_eth_net_offset, context,
527+
target->mark_value);
528+
}
529+
}
530+
531+
return TC_ACT_OK;
532+
#undef BPF_LOG_TOPIC
533+
}
534+
483535
// ================================
484536
// LAN Route Egress
485537
// ================================
@@ -533,6 +585,12 @@ int lan_route_ingress(struct __sk_buff *skb) {
533585
// }
534586
// }
535587

588+
ret = search_route_in_lan(skb, current_eth_net_offset, &context, &flow_mark);
589+
if (ret != TC_ACT_OK) {
590+
skb->mark = replace_flow_source(flow_mark, FLOW_FROM_LAN);
591+
return ret;
592+
}
593+
536594
ret = lan_redirect_check(skb, current_eth_net_offset, &context);
537595
if (ret != TC_ACT_OK) {
538596
return ret;
@@ -547,8 +605,11 @@ int lan_route_ingress(struct __sk_buff *skb) {
547605
skb->mark = replace_flow_source(flow_mark, FLOW_FROM_LAN);
548606

549607
ret = pick_wan_and_send_by_flow_id(skb, current_eth_net_offset, &context, flow_mark);
550-
return ret;
551608

609+
if (ret == TC_ACT_REDIRECT) {
610+
setting_cache_in_lan(&context, flow_mark);
611+
}
612+
return ret;
552613
#undef BPF_LOG_TOPIC
553614
}
554615

@@ -583,11 +644,15 @@ int wan_route_ingress(struct __sk_buff *skb) {
583644
}
584645

585646
ret = lan_redirect_check(skb, current_eth_net_offset, &context);
586-
if (ret != TC_ACT_OK) {
587-
return ret;
647+
if (ret == TC_ACT_REDIRECT) {
648+
u8 mark = get_cache_mask(skb->mark);
649+
if (mark == INGRESS_STATIC_MARK) {
650+
// bpf_log_info("get wan ingress mark: %u", mark);
651+
setting_cache_in_wan(&context, skb->ifindex);
652+
}
588653
}
589654

590-
return TC_ACT_UNSPEC;
655+
return ret == TC_ACT_OK ? TC_ACT_UNSPEC : ret;
591656
#undef BPF_LOG_TOPIC
592657
}
593658

‎landscape-ebpf/src/bpf/landscape.bpf.c‎

Lines changed: 61 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -184,17 +184,76 @@ int xdp_pass(struct xdp_md *ctx) {
184184
#undef BPF_LOG_TOPIC
185185
}
186186

187+
// struct test_item {
188+
// u64 key;
189+
// };
190+
191+
// struct {
192+
// __uint(type, BPF_MAP_TYPE_LRU_HASH);
193+
// __type(key, u32);
194+
// __type(value, struct test_item);
195+
// __uint(max_entries, 1024);
196+
// } test_map SEC(".maps");
197+
187198
SEC("tc")
188199
int mark_egress(struct __sk_buff *skb) {
189200
#define BPF_LOG_TOPIC "<< mark_egress"
190-
201+
u64 time1, time2, time3;
191202
struct ethhdr *eth;
192203
if (VALIDATE_READ_DATA(skb, &eth, 0, sizeof(*eth))) {
193204
return TC_ACT_UNSPEC;
194205
}
195-
196206
bpf_log_info("h_proto: %x", bpf_ntohs(eth->h_proto));
197207

208+
// time1 = bpf_ktime_get_ns();
209+
// time2 = bpf_ktime_get_ns();
210+
// time3 = time2 - time1;
211+
212+
// bpf_log_info("bpf_ktime_get_ns time: %u", time3);
213+
214+
215+
// struct test_item item = {0};
216+
// struct test_item *item2;
217+
// u32 key = 0;
218+
// u64 index = 0;
219+
// u32 num = 3;
220+
221+
// time1 = bpf_ktime_get_ns();
222+
// item2 = bpf_map_lookup_elem(&test_map, &key);
223+
// if (item2 != NULL) {
224+
// __sync_fetch_and_add(&item2->key, 1);
225+
// index = item2->key;
226+
// }
227+
228+
229+
// if (index %num == 0) {
230+
// time1 = bpf_ktime_get_ns();
231+
// bpf_map_update_elem(&test_map, &key, &item, BPF_ANY);
232+
// time2 = bpf_ktime_get_ns();
233+
234+
// bpf_log_info("insert1 time: %u", time2 - time1 - time3);
235+
// } else if (index %num ==1) {
236+
// time1 = bpf_ktime_get_ns();
237+
// item2 = bpf_map_lookup_elem(&test_map, &key);
238+
// if (item2 == NULL) {
239+
// bpf_map_update_elem(&test_map, &key, &item, BPF_ANY);
240+
// }
241+
// time2 = bpf_ktime_get_ns();
242+
243+
// bpf_log_info("insert2 time: %u", time2 - time1 - time3);
244+
// } else if (index %num ==2) {
245+
// time1 = bpf_ktime_get_ns();
246+
// bpf_map_update_elem(&test_map, &key, &item, BPF_NOEXIST );
247+
// time2 = bpf_ktime_get_ns();
248+
249+
// bpf_log_info("insert BPF_NOEXIST time: %u", time2 - time1 - time3);
250+
251+
// }
252+
253+
// time2 = bpf_ktime_get_ns();
254+
// bpf_log_info("__sync_fetch_and_add time: %u", time2 - time1 - time3);
255+
256+
198257
return TC_ACT_UNSPEC;
199258
#undef BPF_LOG_TOPIC
200259
}

0 commit comments

Comments
 (0)