Handle L3 devices (#23)

* Tc bpf can properly dump skb on L3 dev

* Delete obsolete file

bpf/bpf.go

@@ -59,8 +59,8 @@ type BpfConfig struct {
 
 type Objects interface {
 	Load(LoadOptions) error
-	TcIngress() *ebpf.Program
-	TcEgress() *ebpf.Program
+	TcIngress(l2 bool) *ebpf.Program
+	TcEgress(l2 bool) *ebpf.Program
 	Kprobe(pos int) *ebpf.Program
 	Kretprobe() *ebpf.Program
 	KprobeTid() *ebpf.Program
@@ -89,21 +89,24 @@ func (o *Bpf) Load(opts LoadOptions) (err error) {
 		return
 	}
 
-	for _, progName := range []string{"on_ingress", "on_egress"} {
-		prog, ok := o.spec.Programs[progName]
-		if !ok {
-			return errors.Errorf("program %s not found", progName)
-		}
-		if prog.Instructions, err = elibpcap.Inject(
-			opts.Filter,
-			prog.Instructions,
-			elibpcap.Options{
-				AtBpf2Bpf:  "tc_pcap_filter",
-				DirectRead: true,
-				L2Skb:      true,
-			},
-		); err != nil {
-			return
+	for _, suffix := range []string{"l2", "l3"} {
+		for _, progName := range []string{"on_ingress", "on_egress"} {
+			progName = fmt.Sprintf("%s_%s", progName, suffix)
+			prog, ok := o.spec.Programs[progName]
+			if !ok {
+				return errors.Errorf("program %s not found", progName)
+			}
+			if prog.Instructions, err = elibpcap.Inject(
+				opts.Filter,
+				prog.Instructions,
+				elibpcap.Options{
+					AtBpf2Bpf:  "tc_pcap_filter_" + suffix,
+					DirectRead: true,
+					L2Skb:      true,
+				},
+			); err != nil {
+				return
+			}
 		}
 	}
 	for _, progName := range []string{"on_kprobe1", "on_kprobe2", "on_kprobe3", "on_kprobe4", "on_kprobe5"} {
@@ -165,11 +168,17 @@ func (o *Bpf) Load(opts LoadOptions) (err error) {
 	return nil
 }
 
-func (o *Bpf) TcIngress() *ebpf.Program {
-	return o.objs.OnIngress
+func (o *Bpf) TcIngress(l2 bool) *ebpf.Program {
+	if l2 {
+		return o.objs.OnIngressL2
+	}
+	return o.objs.OnIngressL3
 }
-func (o *Bpf) TcEgress() *ebpf.Program {
-	return o.objs.OnEgress
+func (o *Bpf) TcEgress(l2 bool) *ebpf.Program {
+	if l2 {
+		return o.objs.OnEgressL2
+	}
+	return o.objs.OnEgressL3
 }
 
 func (o *Bpf) Kprobe(pos int) *ebpf.Program {

bpf/skbdump.c

@@ -86,22 +86,38 @@ struct bpf_map_def SEC("maps") perf_output = {
 };
 
 static __noinline
-bool tc_pcap_filter(void *_skb, void *__skb, void *___skb, void *data, void* data_end)
+bool tc_pcap_filter_l2(void *_skb, void *__skb, void *___skb, void *data, void* data_end)
+{
+	return data != data_end && _skb == __skb && __skb == ___skb;
+}
+
+static __noinline
+bool tc_pcap_filter_l3(void *_skb, void *__skb, void *___skb, void *data, void* data_end)
 {
 	return data != data_end && _skb == __skb && __skb == ___skb;
 }
 
 static __always_inline
-void handle_skb_tc(struct __sk_buff *skb, bool ingress)
+bool tc_pcap_filter(struct __sk_buff *skb, bool l2)
+{
+	if (l2)
+		return tc_pcap_filter_l2((void *)skb, (void *)skb, (void *)skb,
+					 (void *)(long)skb->data, (void *)(long)skb->data_end);
+
+	return tc_pcap_filter_l3((void *)skb, (void *)skb, (void *)skb,
+				 (void *)(long)skb->data, (void *)(long)skb->data_end);
+}
+
+static __always_inline
+void handle_skb_tc(struct __sk_buff *skb, bool ingress, bool l2)
 {
 	struct skbdump *dump;
 
 	__u64 skb_addr = (__u64)(void *)skb;
 	if (bpf_map_lookup_elem(&skb_addresses, &skb_addr))
 			goto cont;
 
-	if (!tc_pcap_filter((void *)skb, (void *)skb, (void *)skb,
-			 (void *)(long)skb->data, (void *)(long)skb->data_end))
+	if (!tc_pcap_filter(skb, l2))
 		return;
 
 	bpf_map_update_elem(&skb_addresses, &skb_addr, &TRUE, BPF_ANY);
@@ -115,7 +131,7 @@ void handle_skb_tc(struct __sk_buff *skb, bool ingress)
 	dump->meta.time_ns = bpf_ktime_get_boot_ns();
 	dump->meta.skb = skb_addr;
 
-	dump->meta.l2 = 1;
+	dump->meta.l2 = l2;
 	dump->meta.ret = 0;
 	dump->meta.len = skb->len;
 	dump->meta.ifindex = skb->ifindex;
@@ -124,7 +140,7 @@ void handle_skb_tc(struct __sk_buff *skb, bool ingress)
 	__u64 payload_len = dump->meta.len > MAX_PAYLOAD_SIZE ? MAX_PAYLOAD_SIZE : dump->meta.len;
 
 	struct btf_ptr p = {};
-	p.type_id = bpf_core_type_id_kernel(struct __sk_buff);
+	p.type_id = bpf_core_type_id_kernel(struct sk_buff);
 	p.ptr = skb;
 	bpf_snprintf_btf((char *)&dump->meta.structure, MAX_STRUCT_SIZE, &p,
 			 sizeof(p), BTF_F_COMPACT | BTF_F_PTR_RAW);
@@ -134,16 +150,30 @@ void handle_skb_tc(struct __sk_buff *skb, bool ingress)
 }
 
 SEC("tc")
-int on_egress(struct __sk_buff *skb)
+int on_egress_l2(struct __sk_buff *skb)
+{
+	handle_skb_tc(skb, false, true);
+	return TC_ACT_OK;
+}
+
+SEC("tc")
+int on_ingress_l2(struct __sk_buff *skb)
+{
+	handle_skb_tc(skb, true, true);
+	return TC_ACT_OK;
+}
+
+SEC("tc")
+int on_egress_l3(struct __sk_buff *skb)
 {
-	handle_skb_tc(skb, false);
+	handle_skb_tc(skb, false, false);
 	return TC_ACT_OK;
 }
 
 SEC("tc")
-int on_ingress(struct __sk_buff *skb)
+int on_ingress_l3(struct __sk_buff *skb)
 {
-	handle_skb_tc(skb, true);
+	handle_skb_tc(skb, true, false);
 	return TC_ACT_OK;
 }
 
@@ -190,7 +220,7 @@ get_netns(struct sk_buff *skb) {
 }
 
 static __always_inline int
-collect_skb(struct sk_buff *skb, struct pt_regs *ctx, struct skbdump *dump)
+kprobe_collect_skb(struct sk_buff *skb, struct pt_regs *ctx, struct skbdump *dump)
 {
 	dump->meta.time_ns = bpf_ktime_get_boot_ns();
 	dump->meta.skb = (__u64)skb;
@@ -248,7 +278,7 @@ handle_skb_kprobe(struct sk_buff *skb, struct pt_regs *ctx)
 	dump->meta.ret = 0;
 	__u64 sp = ctx->sp;
 	bpf_map_update_elem(&sp2ip, &sp, &dump->meta.at, BPF_ANY);
-	return collect_skb(skb, ctx, dump);
+	return kprobe_collect_skb(skb, ctx, dump);
 }
 
 
@@ -281,7 +311,7 @@ int on_kprobe_tid(struct pt_regs *ctx)
 		__u64 sp = ctx->sp;
 		if (!bpf_map_lookup_elem(&sp2ip, &sp))
 			bpf_map_update_elem(&sp2ip, &sp, &dump->meta.at, BPF_ANY);
-		collect_skb(*skb, ctx, dump);
+		kprobe_collect_skb(*skb, ctx, dump);
 	}
 	return 0;
 }
@@ -305,7 +335,7 @@ int on_kretprobe(struct pt_regs *ctx)
 
 	dump->meta.at = *ip;
 	dump->meta.ret = 1;
-	collect_skb(*skb, ctx, dump);
+	kprobe_collect_skb(*skb, ctx, dump);
 
 	bpf_map_delete_elem(&sp2ip, &sp);
 

go.mod

@@ -4,7 +4,6 @@ go 1.21.0
 
 require (
 	github.com/cilium/ebpf v0.12.3
-	github.com/cloudflare/cbpfc v0.0.0-20230809125630-31aa294050ff
 	github.com/elastic/go-sysinfo v1.9.0
 	github.com/florianl/go-tc v0.4.2
 	github.com/google/gopacket v1.1.19
@@ -13,11 +12,11 @@ require (
 	github.com/spf13/pflag v1.0.5
 	github.com/vishvananda/netlink v1.1.0
 	github.com/vishvananda/netns v0.0.0-20191106174202-0a2b9b5464df
-	golang.org/x/net v0.16.0
 	golang.org/x/sys v0.14.1-0.20231108175955-e4099bfacb8c
 )
 
 require (
+	github.com/cloudflare/cbpfc v0.0.0-20230809125630-31aa294050ff // indirect
 	github.com/elastic/go-windows v1.0.0 // indirect
 	github.com/google/go-cmp v0.5.9 // indirect
 	github.com/joeshaw/multierror v0.0.0-20140124173710-69b34d4ec901 // indirect
@@ -27,6 +26,7 @@ require (
 	github.com/prometheus/procfs v0.8.0 // indirect
 	github.com/stretchr/testify v1.4.0 // indirect
 	golang.org/x/exp v0.0.0-20231110203233-9a3e6036ecaa // indirect
+	golang.org/x/net v0.16.0 // indirect
 	golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4 // indirect
 	gopkg.in/yaml.v2 v2.2.8 // indirect
 	howett.net/plist v0.0.0-20181124034731-591f970eefbb // indirect

target/dev/dev.go

@@ -61,11 +61,11 @@ func (d *Device) Attach(objs bpf.Objects) error {
 		if err = d.EnsureTcQdisc(); err != nil {
 			return
 		}
-		d.delIngress, err = d.AddIngressFilter(objs.TcIngress())
+		d.delIngress, err = d.AddIngressFilter(objs.TcIngress(!d.IsL3Device()))
 		if err != nil {
 			return
 		}
-		d.delEgress, err = d.AddEgressFilter(objs.TcEgress())
+		d.delEgress, err = d.AddEgressFilter(objs.TcEgress(!d.IsL3Device()))
 		return
 	})
 }