--- libaitio/src/bpf.c	2013/06/24 11:09:55	1.1.2.2
+++ libaitio/src/bpf.c	2013/06/26 15:04:02	1.1.2.17
@@ -26,30 +26,90 @@ io_get1stiface(char *szIface, int iflen)
  * io_etherClose() - Close BPF interface
  *
  * @eth = bpf handle
+ * @zcbuf = zero copy buffer, if BPF supports it and isn't NULL
  * return: none
  */
 void
-io_etherClose(int eth)
+io_etherClose(int eth, void **zcbuf)
 {
+#ifdef __FreeBSD__
+	struct bpf_zbuf *zbuf = NULL;
+#endif
+
 	if (eth > STDERR_FILENO)
 		close(eth);
+
+	if (zcbuf && *zcbuf) {
+#ifdef __FreeBSD__
+		zbuf = *zcbuf;
+		munmap(zbuf->bz_bufb, zbuf->bz_buflen);
+		munmap(zbuf->bz_bufa, zbuf->bz_buflen);
+		e_free(*zcbuf);
+		*zcbuf = NULL;
+#endif
+	}
 }
 
+#ifdef __FreeBSD__
+static inline struct bpf_zbuf *
+allocZCbuf(u_int len)
+{
+	struct bpf_zbuf *zbuf = NULL;
+
+	zbuf = e_malloc(sizeof(struct bpf_zbuf));
+	if (!zbuf) {
+		io_SetErr(elwix_GetErrno(), "%s", elwix_GetError());
+		return NULL;
+	} else
+		memset(zbuf, 0, sizeof(struct bpf_zbuf));
+
+	zbuf->bz_buflen = roundup(len, getpagesize());
+	zbuf->bz_bufa = mmap(NULL, zbuf->bz_buflen, PROT_READ | PROT_WRITE, MAP_ANON, -1, 0);
+	if (zbuf->bz_bufa == MAP_FAILED) {
+		LOGERR;
+		e_free(zbuf);
+		return NULL;
+	} else
+		memset(zbuf->bz_bufa, 0, zbuf->bz_buflen);
+	zbuf->bz_bufb = mmap(NULL, zbuf->bz_buflen, PROT_READ | PROT_WRITE, MAP_ANON, -1, 0);
+	if (zbuf->bz_bufb == MAP_FAILED) {
+		LOGERR;
+		munmap(zbuf->bz_bufa, zbuf->bz_buflen);
+		e_free(zbuf);
+		return NULL;
+	} else
+		memset(zbuf->bz_bufb, 0, zbuf->bz_buflen);
+
+	return zbuf;
+}
+#endif
+
 /*
  * io_etherOpen() - Open BPF interface to device
  *
  * @csIface = interface name
  * @flags = open flags
+ * @whdr = with complete headers
+ * @buflen = buffer length
+ * @zcbuf = zero copy buffer, if BPF supports it and isn't NULL
  * return: -1 error or >-1 bpf handle
  */
 int
-io_etherOpen(const char *csIface, int flags)
+io_etherOpen(const char *csIface, int flags, int whdr, u_int *buflen, void **zcbuf)
 {
-	int n = 1, eth = -1;
+	int eth = -1;
 	register int i;
 	char szStr[STRSIZ];
 	struct ifreq ifr;
+	u_int n = 1;
 
+#ifndef __FreeBSD__
+	if (zcbuf) {
+		io_SetErr(ENOTSUP, "bpf zero copy buffer mode is not supported");
+		return -1;
+	}
+#endif
+
 	for (i = 0; i < BPF_DEV_MAX; i++) {
 		memset(szStr, 0, sizeof szStr);
 		snprintf(szStr, sizeof szStr, "/dev/bpf%d", i);
@@ -62,29 +122,191 @@ io_etherOpen(const char *csIface, int flags)
 		return -1;
 	}
 
-	if (ioctl(eth, BIOCIMMEDIATE, &n) == -1) {
-		LOGERR;
-		close(eth);
-		return -1;
+	if (!zcbuf) {
+		if (ioctl(eth, BIOCGBLEN, &n) == -1) {
+			LOGERR;
+			close(eth);
+			return -1;
+		} else
+			n = (buflen && *buflen) ? *buflen : n;
+		if (ioctl(eth, BIOCSBLEN, &n) == -1) {
+			LOGERR;
+			close(eth);
+			return -1;
+		}
+		if (buflen && *buflen)
+			*buflen = n;
+	} else {
+#ifdef __FreeBSD__
+		n = BPF_BUFMODE_ZBUF;
+		if (ioctl(eth, BIOCSETBUFMODE, &n) == -1) {
+			LOGERR;
+			close(eth);
+			return -1;
+		}
+		if (ioctl(eth, BIOCGETZMAX, &n) == -1) {
+			LOGERR;
+			close(eth);
+			return -1;
+		} else
+			n = (buflen && *buflen) ? *buflen : n;
+		if (!(*zcbuf = allocZCbuf(n))) {
+			close(eth);
+			return -1;
+		}
+		if (ioctl(eth, BIOCSETZBUF, *zcbuf) == -1) {
+			LOGERR;
+			io_etherClose(eth, zcbuf);
+			return -1;
+		}
+		if (buflen && *buflen)
+			*buflen = n;
+#endif
 	}
-	n = USHRT_MAX + 1;
-	if (ioctl(eth, BIOCSBLEN, &n) == -1) {
-		LOGERR;
-		close(eth);
-		return -1;
-	}
+
 	if (csIface)
 		strlcpy(szStr, csIface, sizeof szStr);
 	else if (io_get1stiface(szStr, sizeof szStr) == -1) {
-		close(eth);
+		io_etherClose(eth, zcbuf);
 		return -1;
 	}
+	memset(&ifr, 0, sizeof ifr);
 	strlcpy(ifr.ifr_name, szStr, sizeof ifr.ifr_name);
 	if (ioctl(eth, BIOCSETIF, &ifr) == -1) {
 		LOGERR;
-		close(eth);
+		io_etherClose(eth, zcbuf);
 		return -1;
 	}
 
+	n = 1;
+	if (whdr && ioctl(eth, BIOCSHDRCMPLT, &n) == -1) {
+		LOGERR;
+		io_etherClose(eth, zcbuf);
+		return -1;
+	}
+	if (ioctl(eth, BIOCIMMEDIATE, &n) == -1) {
+		LOGERR;
+		io_etherClose(eth, zcbuf);
+		return -1;
+	}
+
 	return eth;
+}
+
+/*
+ * io_etherSend() - Send packet to bpf
+ *
+ * @eth = bpf handle
+ * @buf = buffer
+ * @buflen = buffer length
+ * return: -1 error or !=-1 written bytes
+ */
+ssize_t
+io_etherSend(int eth, const void *buf, size_t buflen)
+{
+	ssize_t wlen = 0;
+
+	if (!buf || !buflen) {
+		io_SetErr(EINVAL, "invalid arguments");
+		return -1;
+	}
+
+	wlen = write(eth, buf, buflen);
+	if (wlen == -1)
+		LOGERR;
+	return wlen;
+}
+
+#ifdef __FreeBSD__
+static inline void
+ackZCbuf(struct bpf_zbuf_header *bzh)
+{
+	atomic_store_rel_int(&bzh->bzh_user_gen, bzh->bzh_kernel_gen);
+}
+
+static inline int
+chkZCbuf(struct bpf_zbuf_header *bzh)
+{
+	/* return true if userspace owns buffer, and false otherwise. */
+	return (bzh->bzh_user_gen != atomic_load_acq_int(&bzh->bzh_kernel_gen));
+}
+
+static inline ssize_t
+nextZCbuf(int eth, void ** __restrict zcache, struct bpf_zbuf * __restrict zbuf)
+{
+	ssize_t rlen = 0;
+	struct bpf_zbuf_header *bzh;
+
+	if (!*zcache || *zcache == zbuf->bz_bufb) {
+		bzh = (struct bpf_zbuf_header *) zbuf->bz_bufa;
+		if (chkZCbuf(bzh)) {
+			*zcache = zbuf->bz_bufa;
+			rlen = atomic_load_acq_int(&bzh->bzh_kernel_len);
+		}
+	} else if (*zcache == zbuf->bz_bufa) {
+		bzh = (struct bpf_zbuf_header *) zbuf->bz_bufb;
+		if (chkZCbuf(bzh)) {
+			*zcache = zbuf->bz_bufb;
+			rlen = atomic_load_acq_int(&bzh->bzh_kernel_len);
+		}
+	}
+
+	return rlen;
+}
+#endif
+
+/*
+ * io_etherRecv() - Receive packet from bpf
+ *
+ * @eth = bpf handle
+ * @buf = buffer
+ * @buflen = buffer length
+ * @zcbuf = zero copy buffer, if BPF supports it and isn't NULL
+ * return: -1 error or !=-1 readed bytes
+ */
+ssize_t
+io_etherRecv(int eth, void * __restrict buf, size_t buflen, void * __restrict zcbuf)
+{
+	ssize_t rlen = 0;
+	void *zcache = NULL;
+#ifdef __FreeBSD__
+	struct bpf_zbuf bz;
+	struct bpf_zbuf_header *bzh;
+#endif
+
+	if (!buf || !buflen) {
+		io_SetErr(EINVAL, "invalid arguments");
+		return -1;
+	}
+
+	if (!zcbuf) {
+		rlen = read(eth, buf, buflen);
+		if (rlen == -1)
+			LOGERR;
+	} else {
+#ifdef __FreeBSD__
+		do {
+			rlen = nextZCbuf(eth, &zcache, (struct bpf_zbuf*) zcbuf);
+			if (rlen > 0) {
+				bzh = (struct bpf_zbuf_header*) zcache;
+				memcpy(buf, zcache + sizeof(struct bpf_zbuf_header), 
+						MIN(buflen, rlen));
+				ackZCbuf(bzh);
+			}
+			if (!rlen && !ioctl(eth, BIOCROTZBUF, &bz)) {
+				bzh = (struct bpf_zbuf_header*) bz.bz_bufa;
+				if (bzh && chkZCbuf(bzh)) {
+					rlen = bz.bz_buflen;
+					memcpy(buf, bz.bz_bufa + sizeof(struct bpf_zbuf_header), 
+							MIN(buflen, rlen));
+				}
+			}
+		} while (0);
+#else
+		rlen = -1;
+		io_SetErr(ENOTSUP, "bpf zero copy buffer mode is not supported");
+#endif
+	}
+
+	return rlen;
 }