--- libaitsched/src/hooks.c 2014/04/27 16:20:37 1.27 +++ libaitsched/src/hooks.c 2022/10/19 01:45:08 1.37 @@ -3,7 +3,7 @@ * by Michael Pounov * * $Author: misho $ -* $Id: hooks.c,v 1.27 2014/04/27 16:20:37 misho Exp $ +* $Id: hooks.c,v 1.37 2022/10/19 01:45:08 misho Exp $ * ************************************************************************** The ELWIX and AITNET software is distributed under the following @@ -12,7 +12,7 @@ terms: All of the documentation and software included in the ELWIX and AITNET Releases is copyrighted by ELWIX - Sofia/Bulgaria -Copyright 2004 - 2014 +Copyright 2004 - 2022 by Michael Pounov . All rights reserved. Redistribution and use in source and binary forms, with or without @@ -47,6 +47,86 @@ SUCH DAMAGE. #include "hooks.h" +static inline void +transit_task2ready(sched_task_t * __restrict t, sched_queue_t * __restrict q) +{ + remove_task_from(t, q); + + t->task_type = taskREADY; + insert_task_to(t, &(TASK_ROOT(t))->root_ready); +} + +#ifdef HAVE_LIBPTHREAD +static void * +_sched_threadWrapper(sched_task_t *t) +{ + void *ret = NULL; + sched_root_task_t *r; + + if (!t || !TASK_ROOT(t)) + pthread_exit(ret); + else + r = (sched_root_task_t*) TASK_ROOT(t); + + pthread_setcancelstate(PTHREAD_CANCEL_ENABLE, NULL); + /* + pthread_setcanceltype(PTHREAD_CANCEL_ASYNCHRONOUS, NULL); + */ + + /* notify parent, thread is ready for execution */ + pthread_testcancel(); + + ret = schedCall(t); + r->root_ret = ret; + + if (TASK_VAL(t)) { + transit_task2unuse(t, &r->root_thread); + TASK_VAL(t) = 0; + } + + pthread_exit(ret); +} +#endif + +#if defined(HAVE_LIBRT) && defined(HAVE_TIMER_CREATE) && \ + defined(HAVE_TIMER_SETTIME) && defined(HAVE_TIMER_DELETE) +#if SUP_ENABLE == KQ_SUPPORT +static void * +_sched_rtcWrapper(sched_task_t *t) +{ + sched_task_t *task; + void *ret; + + if (!t || !TASK_ROOT(t) || !TASK_DATA(t)) + return NULL; + else { + task = (sched_task_t*) TASK_DATA(t); + timer_delete((timer_t) TASK_DATLEN(t)); + } + + ret = schedCall(task); + + transit_task2unuse(task, &(TASK_ROOT(task))->root_rtc); + return ret; +} +#else +static void +_sched_rtcSigWrapper(int sig, siginfo_t *si, void *uc) +{ + sched_task_t *task; + + if (si && si->si_value.sival_ptr) { + task = (sched_task_t*) si->si_value.sival_ptr; + timer_delete((timer_t) TASK_FLAG(task)); + + TASK_RET(task) = (intptr_t) schedCall(task); + + transit_task2unuse(task, &(TASK_ROOT(task))->root_rtc); + } +} +#endif +#endif + /* * sched_hook_init() - Default INIT hook * @@ -62,12 +142,18 @@ sched_hook_init(void *root, void *arg __unused) if (!r) return (void*) -1; -#ifndef KQ_DISABLE +#if SUP_ENABLE == KQ_SUPPORT r->root_kq = kqueue(); if (r->root_kq == -1) { LOGERR; return (void*) -1; } +#elif SUP_ENABLE == EP_SUPPORT + r->root_kq = epoll_create(KQ_EVENTS); + if (r->root_kq == -1) { + LOGERR; + return (void*) -1; + } #else r->root_kq ^= r->root_kq; FD_ZERO(&r->root_fds[0]); @@ -92,7 +178,7 @@ sched_hook_fini(void *root, void *arg __unused) if (!r) return (void*) -1; -#ifndef KQ_DISABLE +#if SUP_ENABLE == KQ_SUPPORT || SUP_ENABLE == EP_SUPPORT if (r->root_kq > 2) { close(r->root_kq); r->root_kq = 0; @@ -116,12 +202,15 @@ sched_hook_fini(void *root, void *arg __unused) void * sched_hook_cancel(void *task, void *arg __unused) { - sched_task_t *t = task; -#ifndef KQ_DISABLE + sched_task_t *t = task, *tmp, *tt; + sched_root_task_t *r = NULL; + int flg = 0; +#if SUP_ENABLE == KQ_SUPPORT struct kevent chg[1]; struct timespec timeout = { 0, 0 }; +#elif SUP_ENABLE == EP_SUPPORT + struct epoll_event ee = { .events = 0, .data.fd = 0 }; #else - sched_root_task_t *r = NULL; register int i; #endif #ifdef AIO_SUPPORT @@ -134,120 +223,188 @@ sched_hook_cancel(void *task, void *arg __unused) if (!t || !TASK_ROOT(t)) return (void*) -1; -#ifdef KQ_DISABLE - r = TASK_ROOT(t); -#endif + else + r = TASK_ROOT(t); switch (TASK_TYPE(t)) { case taskREAD: -#ifndef KQ_DISABLE + /* check for multi subscribers */ + TAILQ_FOREACH_SAFE(tt, &r->root_read, task_node, tmp) + if (TASK_FD(tt) == TASK_FD(t)) + flg++; +#if SUP_ENABLE == KQ_SUPPORT #ifdef __NetBSD__ - EV_SET(&chg[0], TASK_FD(t), EVFILT_READ, EV_DELETE, 0, 0, (intptr_t) TASK_FD(t)); + EV_SET(&chg[0], TASK_FD(t), EVFILT_READ, flg < 2 ? EV_DELETE : 0, + 0, 0, (intptr_t) TASK_FD(t)); #else - EV_SET(&chg[0], TASK_FD(t), EVFILT_READ, EV_DELETE, 0, 0, (void*) TASK_FD(t)); + EV_SET(&chg[0], TASK_FD(t), EVFILT_READ, flg < 2 ? EV_DELETE : 0, + 0, 0, (void*) TASK_FD(t)); #endif +#elif SUP_ENABLE == EP_SUPPORT + ee.data.fd = TASK_FD(t); + ee.events ^= ee.events; + if (FD_ISSET(TASK_FD(t), &r->root_fds[1])) + ee.events = EPOLLOUT; + + if (flg < 2) + FD_CLR(TASK_FD(t), &r->root_fds[0]); + else + ee.events |= EPOLLIN | EPOLLPRI; #else - FD_CLR(TASK_FD(t), &r->root_fds[0]); + if (flg < 2) { + FD_CLR(TASK_FD(t), &r->root_fds[0]); - /* optimize select */ - for (i = r->root_kq - 1; i > 2; i--) - if (FD_ISSET(i, &r->root_fds[0]) || FD_ISSET(i, &r->root_fds[1])) - break; - if (i > 2) + /* optimize select */ + for (i = r->root_kq - 1; i >= 0; i--) + if (FD_ISSET(i, &r->root_fds[0]) || FD_ISSET(i, &r->root_fds[1])) + break; r->root_kq = i + 1; + } #endif break; case taskWRITE: -#ifndef KQ_DISABLE + /* check for multi subscribers */ + TAILQ_FOREACH_SAFE(tt, &r->root_write, task_node, tmp) + if (TASK_FD(tt) == TASK_FD(t)) + flg++; +#if SUP_ENABLE == KQ_SUPPORT #ifdef __NetBSD__ - EV_SET(&chg[0], TASK_FD(t), EVFILT_WRITE, EV_DELETE, 0, 0, (intptr_t) TASK_FD(t)); + EV_SET(&chg[0], TASK_FD(t), EVFILT_WRITE, flg < 2 ? EV_DELETE : 0, + 0, 0, (intptr_t) TASK_FD(t)); #else - EV_SET(&chg[0], TASK_FD(t), EVFILT_WRITE, EV_DELETE, 0, 0, (void*) TASK_FD(t)); + EV_SET(&chg[0], TASK_FD(t), EVFILT_WRITE, flg < 2 ? EV_DELETE : 0, + 0, 0, (void*) TASK_FD(t)); #endif +#elif SUP_ENABLE == EP_SUPPORT + ee.data.fd = TASK_FD(t); + ee.events ^= ee.events; + if (FD_ISSET(TASK_FD(t), &r->root_fds[0])) + ee.events = EPOLLIN | EPOLLPRI; + + if (flg < 2) + FD_CLR(TASK_FD(t), &r->root_fds[1]); + else + ee.events |= EPOLLOUT; #else - FD_CLR(TASK_FD(t), &r->root_fds[1]); + if (flg < 2) { + FD_CLR(TASK_FD(t), &r->root_fds[1]); - /* optimize select */ - for (i = r->root_kq - 1; i > 2; i--) - if (FD_ISSET(i, &r->root_fds[0]) || FD_ISSET(i, &r->root_fds[1])) - break; - if (i > 2) + /* optimize select */ + for (i = r->root_kq - 1; i >= 0; i--) + if (FD_ISSET(i, &r->root_fds[0]) || FD_ISSET(i, &r->root_fds[1])) + break; r->root_kq = i + 1; + } #endif break; case taskALARM: -#ifndef KQ_DISABLE +#if SUP_ENABLE == KQ_SUPPORT + /* check for multi subscribers */ + TAILQ_FOREACH_SAFE(tt, &r->root_alarm, task_node, tmp) + if (TASK_DATA(tt) == TASK_DATA(t)) + flg++; #ifdef __NetBSD__ - EV_SET(&chg[0], (uintptr_t) TASK_DATA(t), EVFILT_TIMER, EV_DELETE, + EV_SET(&chg[0], (uintptr_t) TASK_DATA(t), EVFILT_TIMER, flg < 2 ? EV_DELETE : 0, 0, 0, (intptr_t) TASK_DATA(t)); #else - EV_SET(&chg[0], (uintptr_t) TASK_DATA(t), EVFILT_TIMER, EV_DELETE, + EV_SET(&chg[0], (uintptr_t) TASK_DATA(t), EVFILT_TIMER, flg < 2 ? EV_DELETE : 0, 0, 0, (void*) TASK_DATA(t)); #endif #endif break; case taskNODE: -#ifndef KQ_DISABLE +#if SUP_ENABLE == KQ_SUPPORT + /* check for multi subscribers */ + TAILQ_FOREACH_SAFE(tt, &r->root_node, task_node, tmp) + if (TASK_FD(tt) == TASK_FD(t)) + flg++; #ifdef __NetBSD__ - EV_SET(&chg[0], TASK_FD(t), EVFILT_VNODE, EV_DELETE, 0, 0, (intptr_t) TASK_FD(t)); + EV_SET(&chg[0], TASK_FD(t), EVFILT_VNODE, flg < 2 ? EV_DELETE : 0, + 0, 0, (intptr_t) TASK_FD(t)); #else - EV_SET(&chg[0], TASK_FD(t), EVFILT_VNODE, EV_DELETE, 0, 0, (void*) TASK_FD(t)); + EV_SET(&chg[0], TASK_FD(t), EVFILT_VNODE, flg < 2 ? EV_DELETE : 0, + 0, 0, (void*) TASK_FD(t)); #endif #endif break; case taskPROC: -#ifndef KQ_DISABLE +#if SUP_ENABLE == KQ_SUPPORT + /* check for multi subscribers */ + TAILQ_FOREACH_SAFE(tt, &r->root_proc, task_node, tmp) + if (TASK_VAL(tt) == TASK_VAL(t)) + flg++; #ifdef __NetBSD__ - EV_SET(&chg[0], TASK_VAL(t), EVFILT_PROC, EV_DELETE, 0, 0, (intptr_t) TASK_VAL(t)); + EV_SET(&chg[0], TASK_VAL(t), EVFILT_PROC, flg < 2 ? EV_DELETE : 0, + 0, 0, (intptr_t) TASK_VAL(t)); #else - EV_SET(&chg[0], TASK_VAL(t), EVFILT_PROC, EV_DELETE, 0, 0, (void*) TASK_VAL(t)); + EV_SET(&chg[0], TASK_VAL(t), EVFILT_PROC, flg < 2 ? EV_DELETE : 0, + 0, 0, (void*) TASK_VAL(t)); #endif #endif break; case taskSIGNAL: -#ifndef KQ_DISABLE +#if SUP_ENABLE == KQ_SUPPORT + /* check for multi subscribers */ + TAILQ_FOREACH_SAFE(tt, &r->root_signal, task_node, tmp) + if (TASK_VAL(tt) == TASK_VAL(t)) + flg++; #ifdef __NetBSD__ - EV_SET(&chg[0], TASK_VAL(t), EVFILT_SIGNAL, EV_DELETE, 0, 0, (intptr_t) TASK_VAL(t)); + EV_SET(&chg[0], TASK_VAL(t), EVFILT_SIGNAL, flg < 2 ? EV_DELETE : 0, + 0, 0, (intptr_t) TASK_VAL(t)); #else - EV_SET(&chg[0], TASK_VAL(t), EVFILT_SIGNAL, EV_DELETE, 0, 0, (void*) TASK_VAL(t)); + EV_SET(&chg[0], TASK_VAL(t), EVFILT_SIGNAL, flg < 2 ? EV_DELETE : 0, + 0, 0, (void*) TASK_VAL(t)); #endif /* restore signal */ - signal(TASK_VAL(t), SIG_DFL); + if (flg < 2) + signal(TASK_VAL(t), SIG_DFL); #endif break; #ifdef AIO_SUPPORT case taskAIO: -#ifndef KQ_DISABLE +#if SUP_ENABLE == KQ_SUPPORT + /* check for multi subscribers */ + TAILQ_FOREACH_SAFE(tt, &r->root_aio, task_node, tmp) + if (TASK_VAL(tt) == TASK_VAL(t)) + flg++; #ifdef __NetBSD__ - EV_SET(&chg[0], TASK_VAL(t), EVFILT_AIO, EV_DELETE, 0, 0, (intptr_t) TASK_VAL(t)); + EV_SET(&chg[0], TASK_VAL(t), EVFILT_AIO, flg < 2 ? EV_DELETE : 0, + 0, 0, (intptr_t) TASK_VAL(t)); #else - EV_SET(&chg[0], TASK_VAL(t), EVFILT_AIO, EV_DELETE, 0, 0, (void*) TASK_VAL(t)); + EV_SET(&chg[0], TASK_VAL(t), EVFILT_AIO, flg < 2 ? EV_DELETE : 0, + 0, 0, (void*) TASK_VAL(t)); #endif acb = (struct aiocb*) TASK_VAL(t); if (acb) { if (aio_cancel(acb->aio_fildes, acb) == AIO_CANCELED) aio_return(acb); - free(acb); + e_free(acb); TASK_VAL(t) = 0; } #endif break; #ifdef EVFILT_LIO case taskLIO: -#ifndef KQ_DISABLE +#if SUP_ENABLE == KQ_SUPPORT + /* check for multi subscribers */ + TAILQ_FOREACH_SAFE(tt, &r->root_lio, task_node, tmp) + if (TASK_VAL(tt) == TASK_VAL(t)) + flg++; #ifdef __NetBSD__ - EV_SET(&chg[0], TASK_VAL(t), EVFILT_LIO, EV_DELETE, 0, 0, (intptr_t) TASK_VAL(t)); + EV_SET(&chg[0], TASK_VAL(t), EVFILT_LIO, flg < 2 ? EV_DELETE : 0, + 0, 0, (intptr_t) TASK_VAL(t)); #else - EV_SET(&chg[0], TASK_VAL(t), EVFILT_LIO, EV_DELETE, 0, 0, (void*) TASK_VAL(t)); + EV_SET(&chg[0], TASK_VAL(t), EVFILT_LIO, flg < 2 ? EV_DELETE : 0, + 0, 0, (void*) TASK_VAL(t)); #endif acbs = (struct aiocb**) TASK_VAL(t); if (acbs) { for (i = 0; i < TASK_DATLEN(t); i++) { if (aio_cancel(acbs[i]->aio_fildes, acbs[i]) == AIO_CANCELED) aio_return(acbs[i]); - free(acbs[i]); + e_free(acbs[i]); } - free(acbs); + e_free(acbs); TASK_VAL(t) = 0; } #endif @@ -256,32 +413,61 @@ sched_hook_cancel(void *task, void *arg __unused) #endif /* AIO_SUPPORT */ #ifdef EVFILT_USER case taskUSER: -#ifndef KQ_DISABLE +#if SUP_ENABLE == KQ_SUPPORT + /* check for multi subscribers */ + TAILQ_FOREACH_SAFE(tt, &r->root_user, task_node, tmp) + if (TASK_VAL(tt) == TASK_VAL(t)) + flg++; #ifdef __NetBSD__ - EV_SET(&chg[0], TASK_VAL(t), EVFILT_USER, EV_DELETE, 0, 0, (intptr_t) TASK_VAL(t)); + EV_SET(&chg[0], TASK_VAL(t), EVFILT_USER, flg < 2 ? EV_DELETE : 0, + 0, 0, (intptr_t) TASK_VAL(t)); #else - EV_SET(&chg[0], TASK_VAL(t), EVFILT_USER, EV_DELETE, 0, 0, (void*) TASK_VAL(t)); + EV_SET(&chg[0], TASK_VAL(t), EVFILT_USER, flg < 2 ? EV_DELETE : 0, + 0, 0, (void*) TASK_VAL(t)); #endif #endif break; #endif /* EVFILT_USER */ case taskTHREAD: #ifdef HAVE_LIBPTHREAD - pthread_cancel((pthread_t) TASK_VAL(t)); + if (TASK_VAL(t)) { + pthread_cancel((pthread_t) TASK_VAL(t)); + pthread_join((pthread_t) TASK_VAL(t), NULL); + if (TASK_VAL(t)) { + transit_task2unuse(t, &(TASK_ROOT(t))->root_thread); + TASK_VAL(t) = 0; + } + } #endif return NULL; -#if defined(HAVE_TIMER_CREATE) && defined(HAVE_TIMER_SETTIME) && defined(HAVE_TIMER_DELETE) +#if defined(HAVE_LIBRT) && defined(HAVE_TIMER_CREATE) && \ + defined(HAVE_TIMER_SETTIME) && defined(HAVE_TIMER_DELETE) case taskRTC: timer_delete((timer_t) TASK_FLAG(t)); +#if SUP_ENABLE == KQ_SUPPORT schedCancel((sched_task_t*) TASK_RET(t)); +#else + /* check for multi subscribers */ + TAILQ_FOREACH_SAFE(tt, &r->root_rtc, task_node, tmp) + if (TASK_DATA(tt) == TASK_DATA(t)) + flg++; + + /* restore signal */ + if (flg < 2) + signal((intptr_t) TASK_DATA(t) + SIGRTMIN, SIG_DFL); +#endif return NULL; #endif /* HAVE_TIMER_CREATE */ default: return NULL; } -#ifndef KQ_DISABLE - kevent(TASK_ROOT(t)->root_kq, chg, 1, NULL, 0, &timeout); +#if SUP_ENABLE == KQ_SUPPORT + kevent(r->root_kq, chg, 1, NULL, 0, &timeout); +#elif SUP_ENABLE == EP_SUPPORT + if (TASK_TYPE(t) == taskREAD || TASK_TYPE(t) == taskWRITE) { + epoll_ctl(r->root_kq, ee.events ? EPOLL_CTL_MOD : EPOLL_CTL_DEL, ee.data.fd, &ee); + } #endif return NULL; } @@ -306,10 +492,12 @@ sched_hook_thread(void *task, void *arg) sigfillset(&s); pthread_sigmask(SIG_BLOCK, &s, &o); - if ((errno = pthread_create(&tid, (pthread_attr_t*) arg, - (void *(*)(void*)) _sched_threadWrapper, t))) { + errno = pthread_create(&tid, (pthread_attr_t*) arg, + (void *(*)(void*)) _sched_threadWrapper, t); + pthread_sigmask(SIG_SETMASK, &o, NULL); + + if (errno) { LOGERR; - pthread_sigmask(SIG_SETMASK, &o, NULL); return (void*) -1; } else TASK_VAL(t) = (u_long) tid; @@ -317,7 +505,6 @@ sched_hook_thread(void *task, void *arg) if (!TASK_ISLOCKED(t)) TASK_LOCK(t); - pthread_sigmask(SIG_SETMASK, &o, NULL); return NULL; } #endif @@ -333,32 +520,51 @@ void * sched_hook_read(void *task, void *arg __unused) { sched_task_t *t = task; -#ifndef KQ_DISABLE + sched_root_task_t *r = NULL; +#if SUP_ENABLE == KQ_SUPPORT struct kevent chg[1]; struct timespec timeout = { 0, 0 }; -#else - sched_root_task_t *r = NULL; +#elif SUP_ENABLE == EP_SUPPORT + struct epoll_event ee; + int flg = 0; #endif if (!t || !TASK_ROOT(t)) return (void*) -1; -#ifdef KQ_DISABLE - r = TASK_ROOT(t); -#endif + else + r = TASK_ROOT(t); -#ifndef KQ_DISABLE +#if SUP_ENABLE == KQ_SUPPORT #ifdef __NetBSD__ EV_SET(&chg[0], TASK_FD(t), EVFILT_READ, EV_ADD | EV_CLEAR, 0, 0, (intptr_t) TASK_FD(t)); #else EV_SET(&chg[0], TASK_FD(t), EVFILT_READ, EV_ADD | EV_CLEAR, 0, 0, (void*) TASK_FD(t)); #endif - if (kevent(TASK_ROOT(t)->root_kq, chg, 1, NULL, 0, &timeout) == -1) { - if (TASK_ROOT(t)->root_hooks.hook_exec.exception) - TASK_ROOT(t)->root_hooks.hook_exec.exception(TASK_ROOT(t), NULL); + if (kevent(r->root_kq, chg, 1, NULL, 0, &timeout) == -1) { + if (r->root_hooks.hook_exec.exception) + r->root_hooks.hook_exec.exception(r, NULL); else LOGERR; return (void*) -1; } +#elif SUP_ENABLE == EP_SUPPORT + ee.data.fd = TASK_FD(t); + ee.events = EPOLLIN | EPOLLPRI; + if (FD_ISSET(TASK_FD(t), &r->root_fds[0])) + flg |= 1; + if (FD_ISSET(TASK_FD(t), &r->root_fds[1])) { + flg |= 2; + ee.events |= EPOLLOUT; + } + + if (epoll_ctl(r->root_kq, flg ? EPOLL_CTL_MOD : EPOLL_CTL_ADD, TASK_FD(t), &ee) == -1) { + if (r->root_hooks.hook_exec.exception) + r->root_hooks.hook_exec.exception(r, NULL); + else + LOGERR; + return (void*) -1; + } else + FD_SET(TASK_FD(t), &r->root_fds[0]); #else FD_SET(TASK_FD(t), &r->root_fds[0]); if (TASK_FD(t) >= r->root_kq) @@ -379,32 +585,53 @@ void * sched_hook_write(void *task, void *arg __unused) { sched_task_t *t = task; -#ifndef KQ_DISABLE + sched_root_task_t *r = NULL; +#if SUP_ENABLE == KQ_SUPPORT struct kevent chg[1]; struct timespec timeout = { 0, 0 }; -#else - sched_root_task_t *r = NULL; +#elif SUP_ENABLE == EP_SUPPORT + struct epoll_event ee; + int flg = 0; #endif if (!t || !TASK_ROOT(t)) return (void*) -1; -#ifdef KQ_DISABLE - r = TASK_ROOT(t); -#endif + else + r = TASK_ROOT(t); -#ifndef KQ_DISABLE +#if SUP_ENABLE == KQ_SUPPORT #ifdef __NetBSD__ EV_SET(&chg[0], TASK_FD(t), EVFILT_WRITE, EV_ADD | EV_CLEAR, 0, 0, (intptr_t) TASK_FD(t)); #else EV_SET(&chg[0], TASK_FD(t), EVFILT_WRITE, EV_ADD | EV_CLEAR, 0, 0, (void*) TASK_FD(t)); #endif - if (kevent(TASK_ROOT(t)->root_kq, chg, 1, NULL, 0, &timeout) == -1) { - if (TASK_ROOT(t)->root_hooks.hook_exec.exception) - TASK_ROOT(t)->root_hooks.hook_exec.exception(TASK_ROOT(t), NULL); + if (kevent(r->root_kq, chg, 1, NULL, 0, &timeout) == -1) { + if (r->root_hooks.hook_exec.exception) + r->root_hooks.hook_exec.exception(r, NULL); else LOGERR; return (void*) -1; } +#elif SUP_ENABLE == EP_SUPPORT + ee.data.fd = TASK_FD(t); + ee.events = EPOLLOUT; + + if (FD_ISSET(TASK_FD(t), &r->root_fds[0])) { + flg |= 1; + ee.events |= EPOLLIN | EPOLLPRI; + } + if (FD_ISSET(TASK_FD(t), &r->root_fds[1])) { + flg |= 2; + } + + if (epoll_ctl(r->root_kq, flg ? EPOLL_CTL_MOD : EPOLL_CTL_ADD, TASK_FD(t), &ee) == -1) { + if (r->root_hooks.hook_exec.exception) + r->root_hooks.hook_exec.exception(r, NULL); + else + LOGERR; + return (void*) -1; + } else + FD_SET(TASK_FD(t), &r->root_fds[1]); #else FD_SET(TASK_FD(t), &r->root_fds[1]); if (TASK_FD(t) >= r->root_kq) @@ -424,7 +651,7 @@ sched_hook_write(void *task, void *arg __unused) void * sched_hook_alarm(void *task, void *arg __unused) { -#ifndef KQ_DISABLE +#if SUP_ENABLE == KQ_SUPPORT sched_task_t *t = task; struct kevent chg[1]; struct timespec timeout = { 0, 0 }; @@ -457,16 +684,17 @@ sched_hook_alarm(void *task, void *arg __unused) * sched_hook_node() - Default NODE hook * * @task = current task - * @arg = unused + * @arg = if arg == 42 then waiting for all events * return: <0 errors and 0 ok */ void * -sched_hook_node(void *task, void *arg __unused) +sched_hook_node(void *task, void *arg) { -#ifndef KQ_DISABLE +#if SUP_ENABLE == KQ_SUPPORT sched_task_t *t = task; struct kevent chg[1]; struct timespec timeout = { 0, 0 }; + u_int addflags = (u_int) (uintptr_t) arg; if (!t || !TASK_ROOT(t)) return (void*) -1; @@ -474,11 +702,11 @@ sched_hook_node(void *task, void *arg __unused) #ifdef __NetBSD__ EV_SET(&chg[0], TASK_FD(t), EVFILT_VNODE, EV_ADD | EV_CLEAR, NOTE_DELETE | NOTE_WRITE | NOTE_EXTEND | NOTE_ATTRIB | - NOTE_LINK | NOTE_RENAME | NOTE_REVOKE, 0, (intptr_t) TASK_FD(t)); + NOTE_LINK | NOTE_RENAME | NOTE_REVOKE | addflags, 0, (intptr_t) TASK_FD(t)); #else EV_SET(&chg[0], TASK_FD(t), EVFILT_VNODE, EV_ADD | EV_CLEAR, NOTE_DELETE | NOTE_WRITE | NOTE_EXTEND | NOTE_ATTRIB | - NOTE_LINK | NOTE_RENAME | NOTE_REVOKE, 0, (void*) TASK_FD(t)); + NOTE_LINK | NOTE_RENAME | NOTE_REVOKE | addflags, 0, (void*) TASK_FD(t)); #endif if (kevent(TASK_ROOT(t)->root_kq, chg, 1, NULL, 0, &timeout) == -1) { if (TASK_ROOT(t)->root_hooks.hook_exec.exception) @@ -502,7 +730,7 @@ sched_hook_node(void *task, void *arg __unused) void * sched_hook_proc(void *task, void *arg __unused) { -#ifndef KQ_DISABLE +#if SUP_ENABLE == KQ_SUPPORT sched_task_t *t = task; struct kevent chg[1]; struct timespec timeout = { 0, 0 }; @@ -539,7 +767,7 @@ sched_hook_proc(void *task, void *arg __unused) void * sched_hook_signal(void *task, void *arg __unused) { -#ifndef KQ_DISABLE +#if SUP_ENABLE == KQ_SUPPORT sched_task_t *t = task; struct kevent chg[1]; struct timespec timeout = { 0, 0 }; @@ -562,24 +790,6 @@ sched_hook_signal(void *task, void *arg __unused) LOGERR; return (void*) -1; } -#else -#if 0 - sched_task_t *t = task; - struct sigaction sa; - - memset(&sa, 0, sizeof sa); - sigemptyset(&sa.sa_mask); - sa.sa_handler = _sched_sigHandler; - sa.sa_flags = SA_RESETHAND | SA_RESTART; - - if (sigaction(TASK_VAL(t), &sa, NULL) == -1) { - if (TASK_ROOT(t)->root_hooks.hook_exec.exception) - TASK_ROOT(t)->root_hooks.hook_exec.exception(TASK_ROOT(t), NULL); - else - LOGERR; - return (void*) -1; - } -#endif /* 0 */ #endif return NULL; } @@ -595,7 +805,7 @@ sched_hook_signal(void *task, void *arg __unused) void * sched_hook_user(void *task, void *arg __unused) { -#ifndef KQ_DISABLE +#if SUP_ENABLE == KQ_SUPPORT sched_task_t *t = task; struct kevent chg[1]; struct timespec timeout = { 0, 0 }; @@ -623,664 +833,571 @@ sched_hook_user(void *task, void *arg __unused) } #endif -/* - * sched_hook_fetch() - Default FETCH hook - * - * @root = root task - * @arg = unused - * return: NULL error or !=NULL fetched task - */ -void * -sched_hook_fetch(void *root, void *arg __unused) +#if SUP_ENABLE == KQ_SUPPORT +static inline void +fetch_hook_kevent_proceed(int en, struct kevent *res, sched_root_task_t *r) { - sched_root_task_t *r = root; - sched_task_t *task, *tmp; - struct timespec now, m, mtmp; -#ifndef KQ_DISABLE - struct kevent evt[1], res[KQ_EVENTS]; - struct timespec *timeout; -#else - struct timeval *timeout, tv; - fd_set rfd, wfd, xfd; -#endif + struct kevent evt[1]; register int i, flg; - int en; + sched_task_t *task, *tmp; + struct timespec now = { 0, 0 }; #ifdef AIO_SUPPORT int len, fd; struct aiocb *acb; #ifdef EVFILT_LIO int l; - register int j; off_t off; struct aiocb **acbs; struct iovec *iv; #endif /* EVFILT_LIO */ #endif /* AIO_SUPPORT */ - if (!r) - return NULL; - - /* get new task by queue priority */ - while ((task = TAILQ_FIRST(&r->root_event))) { -#ifdef HAVE_LIBPTHREAD - pthread_mutex_lock(&r->root_mtx[taskEVENT]); -#endif - TAILQ_REMOVE(&r->root_event, task, task_node); -#ifdef HAVE_LIBPTHREAD - pthread_mutex_unlock(&r->root_mtx[taskEVENT]); -#endif - task->task_type = taskUNUSE; -#ifdef HAVE_LIBPTHREAD - pthread_mutex_lock(&r->root_mtx[taskUNUSE]); -#endif - TAILQ_INSERT_TAIL(&r->root_unuse, task, task_node); -#ifdef HAVE_LIBPTHREAD - pthread_mutex_unlock(&r->root_mtx[taskUNUSE]); -#endif - return task; - } - while ((task = TAILQ_FIRST(&r->root_ready))) { -#ifdef HAVE_LIBPTHREAD - pthread_mutex_lock(&r->root_mtx[taskREADY]); -#endif - TAILQ_REMOVE(&r->root_ready, task, task_node); -#ifdef HAVE_LIBPTHREAD - pthread_mutex_unlock(&r->root_mtx[taskREADY]); -#endif - task->task_type = taskUNUSE; -#ifdef HAVE_LIBPTHREAD - pthread_mutex_lock(&r->root_mtx[taskUNUSE]); -#endif - TAILQ_INSERT_TAIL(&r->root_unuse, task, task_node); -#ifdef HAVE_LIBPTHREAD - pthread_mutex_unlock(&r->root_mtx[taskUNUSE]); -#endif - return task; - } - -#ifdef TIMER_WITHOUT_SORT - clock_gettime(CLOCK_MONOTONIC, &now); - - sched_timespecclear(&r->root_wait); - TAILQ_FOREACH(task, &r->root_timer, task_node) { - if (!sched_timespecisset(&r->root_wait)) - r->root_wait = TASK_TS(task); - else if (sched_timespeccmp(&TASK_TS(task), &r->root_wait, -) < 0) - r->root_wait = TASK_TS(task); - } - - if (TAILQ_FIRST(&r->root_timer)) { - m = r->root_wait; - sched_timespecsub(&m, &now, &mtmp); - r->root_wait = mtmp; - } else { - /* set wait INFTIM */ - sched_timespecinf(&r->root_wait); - } -#else /* ! TIMER_WITHOUT_SORT */ - if (!TAILQ_FIRST(&r->root_task) && (task = TAILQ_FIRST(&r->root_timer))) { - clock_gettime(CLOCK_MONOTONIC, &now); - - m = TASK_TS(task); - sched_timespecsub(&m, &now, &mtmp); - r->root_wait = mtmp; - } else { - /* set wait INFTIM */ - sched_timespecinf(&r->root_wait); - } -#endif /* TIMER_WITHOUT_SORT */ - /* if present member of task, set NOWAIT */ - if (TAILQ_FIRST(&r->root_task)) - sched_timespecclear(&r->root_wait); - - if (r->root_wait.tv_sec != -1 && r->root_wait.tv_nsec != -1) { -#ifndef KQ_DISABLE - timeout = &r->root_wait; -#else - sched_timespec2val(&r->root_wait, &tv); - timeout = &tv; -#endif /* KQ_DISABLE */ - } else if (sched_timespecisinf(&r->root_poll)) - timeout = NULL; - else { -#ifndef KQ_DISABLE - timeout = &r->root_poll; -#else - sched_timespec2val(&r->root_poll, &tv); - timeout = &tv; -#endif /* KQ_DISABLE */ - } - -#ifndef KQ_DISABLE - if ((en = kevent(r->root_kq, NULL, 0, res, KQ_EVENTS, timeout)) == -1) { -#else - rfd = xfd = r->root_fds[0]; - wfd = r->root_fds[1]; - if ((en = select(r->root_kq, &rfd, &wfd, &xfd, timeout)) == -1) { -#endif /* KQ_DISABLE */ - if (r->root_hooks.hook_exec.exception) { - if (r->root_hooks.hook_exec.exception(r, NULL)) - return NULL; - } else if (errno != EINTR) - LOGERR; - goto skip_event; - } - - /* kevent dispatcher */ - now.tv_sec = now.tv_nsec = 0; - /* Go and catch the cat into pipes ... */ -#ifndef KQ_DISABLE for (i = 0; i < en; i++) { memcpy(evt, &res[i], sizeof evt); evt->flags = EV_DELETE; /* Put read/write task to ready queue */ + flg = 0; switch (res[i].filter) { case EVFILT_READ: - flg = 0; TAILQ_FOREACH_SAFE(task, &r->root_read, task_node, tmp) { - if (TASK_FD(task) != ((intptr_t) res[i].udata)) - continue; - else { + if (TASK_FD(task) == ((intptr_t) res[i].udata)) { + if (!flg) { + TASK_RET(task) = res[i].data; + TASK_FLAG(task) = (u_long) res[i].fflags; + + /* remove read handle */ + remove_task_from(task, &r->root_read); + + if (r->root_hooks.hook_exec.exception && res[i].flags & EV_EOF) { + if (r->root_hooks.hook_exec.exception(r, (void*) EV_EOF)) { + task->task_type = taskUNUSE; + insert_task_to(task, &r->root_unuse); + } else { + task->task_type = taskREADY; + insert_task_to(task, &r->root_ready); + } + } else { + task->task_type = taskREADY; + insert_task_to(task, &r->root_ready); + } + } flg++; - TASK_RET(task) = res[i].data; - TASK_FLAG(task) = (u_long) res[i].fflags; } - /* remove read handle */ -#ifdef HAVE_LIBPTHREAD - pthread_mutex_lock(&r->root_mtx[taskREAD]); -#endif - TAILQ_REMOVE(&r->root_read, task, task_node); -#ifdef HAVE_LIBPTHREAD - pthread_mutex_unlock(&r->root_mtx[taskREAD]); -#endif - if (r->root_hooks.hook_exec.exception && res[i].flags & EV_EOF) { - if (r->root_hooks.hook_exec.exception(r, (void*) EV_EOF)) { - task->task_type = taskUNUSE; -#ifdef HAVE_LIBPTHREAD - pthread_mutex_lock(&r->root_mtx[taskUNUSE]); -#endif - TAILQ_INSERT_TAIL(&r->root_unuse, task, task_node); -#ifdef HAVE_LIBPTHREAD - pthread_mutex_unlock(&r->root_mtx[taskUNUSE]); -#endif - } else { - task->task_type = taskREADY; -#ifdef HAVE_LIBPTHREAD - pthread_mutex_lock(&r->root_mtx[taskREADY]); -#endif - TAILQ_INSERT_TAIL(&r->root_ready, task, task_node); -#ifdef HAVE_LIBPTHREAD - pthread_mutex_unlock(&r->root_mtx[taskREADY]); -#endif - } - } else { - task->task_type = taskREADY; -#ifdef HAVE_LIBPTHREAD - pthread_mutex_lock(&r->root_mtx[taskREADY]); -#endif - TAILQ_INSERT_TAIL(&r->root_ready, task, task_node); -#ifdef HAVE_LIBPTHREAD - pthread_mutex_unlock(&r->root_mtx[taskREADY]); -#endif - } } - /* if match at least 2, don't remove resouce of event */ - if (flg > 1) - evt->flags ^= evt->flags; break; case EVFILT_WRITE: - flg = 0; TAILQ_FOREACH_SAFE(task, &r->root_write, task_node, tmp) { - if (TASK_FD(task) != ((intptr_t) res[i].udata)) - continue; - else { + if (TASK_FD(task) == ((intptr_t) res[i].udata)) { + if (!flg) { + TASK_RET(task) = res[i].data; + TASK_FLAG(task) = (u_long) res[i].fflags; + + /* remove write handle */ + remove_task_from(task, &r->root_write); + + if (r->root_hooks.hook_exec.exception && res[i].flags & EV_EOF) { + if (r->root_hooks.hook_exec.exception(r, (void*) EV_EOF)) { + task->task_type = taskUNUSE; + insert_task_to(task, &r->root_unuse); + } else { + task->task_type = taskREADY; + insert_task_to(task, &r->root_ready); + } + } else { + task->task_type = taskREADY; + insert_task_to(task, &r->root_ready); + } + } flg++; - TASK_RET(task) = res[i].data; - TASK_FLAG(task) = (u_long) res[i].fflags; } - /* remove write handle */ -#ifdef HAVE_LIBPTHREAD - pthread_mutex_lock(&r->root_mtx[taskWRITE]); -#endif - TAILQ_REMOVE(&r->root_write, task, task_node); -#ifdef HAVE_LIBPTHREAD - pthread_mutex_unlock(&r->root_mtx[taskWRITE]); -#endif - if (r->root_hooks.hook_exec.exception && res[i].flags & EV_EOF) { - if (r->root_hooks.hook_exec.exception(r, (void*) EV_EOF)) { - task->task_type = taskUNUSE; -#ifdef HAVE_LIBPTHREAD - pthread_mutex_lock(&r->root_mtx[taskUNUSE]); -#endif - TAILQ_INSERT_TAIL(&r->root_unuse, task, task_node); -#ifdef HAVE_LIBPTHREAD - pthread_mutex_unlock(&r->root_mtx[taskUNUSE]); -#endif - } else { - task->task_type = taskREADY; -#ifdef HAVE_LIBPTHREAD - pthread_mutex_lock(&r->root_mtx[taskREADY]); -#endif - TAILQ_INSERT_TAIL(&r->root_ready, task, task_node); -#ifdef HAVE_LIBPTHREAD - pthread_mutex_unlock(&r->root_mtx[taskREADY]); -#endif - } - } else { - task->task_type = taskREADY; -#ifdef HAVE_LIBPTHREAD - pthread_mutex_lock(&r->root_mtx[taskREADY]); -#endif - TAILQ_INSERT_TAIL(&r->root_ready, task, task_node); -#ifdef HAVE_LIBPTHREAD - pthread_mutex_unlock(&r->root_mtx[taskREADY]); -#endif - } } - /* if match at least 2, don't remove resouce of event */ - if (flg > 1) - evt->flags ^= evt->flags; break; case EVFILT_TIMER: - flg = 0; TAILQ_FOREACH_SAFE(task, &r->root_alarm, task_node, tmp) { - if ((uintptr_t) TASK_DATA(task) != ((uintptr_t) res[i].udata)) - continue; - else { + if ((uintptr_t) TASK_DATA(task) == ((uintptr_t) res[i].udata)) { + if (!flg) { + TASK_RET(task) = res[i].data; + TASK_FLAG(task) = (u_long) res[i].fflags; + + /* remove alarm handle */ + transit_task2ready(task, &r->root_alarm); + } flg++; - TASK_RET(task) = res[i].data; - TASK_FLAG(task) = (u_long) res[i].fflags; } - /* remove alarm handle */ -#ifdef HAVE_LIBPTHREAD - pthread_mutex_lock(&r->root_mtx[taskALARM]); -#endif - TAILQ_REMOVE(&r->root_alarm, task, task_node); -#ifdef HAVE_LIBPTHREAD - pthread_mutex_unlock(&r->root_mtx[taskALARM]); -#endif - task->task_type = taskREADY; -#ifdef HAVE_LIBPTHREAD - pthread_mutex_lock(&r->root_mtx[taskREADY]); -#endif - TAILQ_INSERT_TAIL(&r->root_ready, task, task_node); -#ifdef HAVE_LIBPTHREAD - pthread_mutex_unlock(&r->root_mtx[taskREADY]); -#endif } - /* if match at least 2, don't remove resouce of event */ - if (flg > 1) - evt->flags ^= evt->flags; break; case EVFILT_VNODE: - flg = 0; TAILQ_FOREACH_SAFE(task, &r->root_node, task_node, tmp) { - if (TASK_FD(task) != ((intptr_t) res[i].udata)) - continue; - else { + if (TASK_FD(task) == ((intptr_t) res[i].udata)) { + if (!flg) { + TASK_RET(task) = res[i].data; + TASK_FLAG(task) = (u_long) res[i].fflags; + + /* remove node handle */ + transit_task2ready(task, &r->root_node); + } flg++; - TASK_RET(task) = res[i].data; - TASK_FLAG(task) = (u_long) res[i].fflags; } - /* remove node handle */ -#ifdef HAVE_LIBPTHREAD - pthread_mutex_lock(&r->root_mtx[taskNODE]); -#endif - TAILQ_REMOVE(&r->root_node, task, task_node); -#ifdef HAVE_LIBPTHREAD - pthread_mutex_unlock(&r->root_mtx[taskNODE]); -#endif - task->task_type = taskREADY; -#ifdef HAVE_LIBPTHREAD - pthread_mutex_lock(&r->root_mtx[taskREADY]); -#endif - TAILQ_INSERT_TAIL(&r->root_ready, task, task_node); -#ifdef HAVE_LIBPTHREAD - pthread_mutex_unlock(&r->root_mtx[taskREADY]); -#endif } - /* if match at least 2, don't remove resouce of event */ - if (flg > 1) - evt->flags ^= evt->flags; break; case EVFILT_PROC: - flg = 0; TAILQ_FOREACH_SAFE(task, &r->root_proc, task_node, tmp) { - if (TASK_VAL(task) != ((uintptr_t) res[i].udata)) - continue; - else { + if (TASK_VAL(task) == ((uintptr_t) res[i].udata)) { + if (!flg) { + TASK_RET(task) = res[i].data; + TASK_FLAG(task) = (u_long) res[i].fflags; + + /* remove proc handle */ + transit_task2ready(task, &r->root_proc); + } flg++; - TASK_RET(task) = res[i].data; - TASK_FLAG(task) = (u_long) res[i].fflags; } - /* remove proc handle */ -#ifdef HAVE_LIBPTHREAD - pthread_mutex_lock(&r->root_mtx[taskPROC]); -#endif - TAILQ_REMOVE(&r->root_proc, task, task_node); -#ifdef HAVE_LIBPTHREAD - pthread_mutex_unlock(&r->root_mtx[taskPROC]); -#endif - task->task_type = taskREADY; -#ifdef HAVE_LIBPTHREAD - pthread_mutex_lock(&r->root_mtx[taskREADY]); -#endif - TAILQ_INSERT_TAIL(&r->root_ready, task, task_node); -#ifdef HAVE_LIBPTHREAD - pthread_mutex_unlock(&r->root_mtx[taskREADY]); -#endif } - /* if match at least 2, don't remove resouce of event */ - if (flg > 1) - evt->flags ^= evt->flags; break; case EVFILT_SIGNAL: - flg = 0; TAILQ_FOREACH_SAFE(task, &r->root_signal, task_node, tmp) { - if (TASK_VAL(task) != ((uintptr_t) res[i].udata)) - continue; - else { + if (TASK_VAL(task) == ((uintptr_t) res[i].udata)) { + if (!flg) { + TASK_RET(task) = res[i].data; + TASK_FLAG(task) = (u_long) res[i].fflags; + + /* remove signal handle */ + transit_task2ready(task, &r->root_signal); + } flg++; - TASK_RET(task) = res[i].data; - TASK_FLAG(task) = (u_long) res[i].fflags; } - /* remove signal handle */ -#ifdef HAVE_LIBPTHREAD - pthread_mutex_lock(&r->root_mtx[taskSIGNAL]); -#endif - TAILQ_REMOVE(&r->root_signal, task, task_node); -#ifdef HAVE_LIBPTHREAD - pthread_mutex_unlock(&r->root_mtx[taskSIGNAL]); -#endif - task->task_type = taskREADY; -#ifdef HAVE_LIBPTHREAD - pthread_mutex_lock(&r->root_mtx[taskREADY]); -#endif - TAILQ_INSERT_TAIL(&r->root_ready, task, task_node); -#ifdef HAVE_LIBPTHREAD - pthread_mutex_unlock(&r->root_mtx[taskREADY]); -#endif } - /* if match at least 2, don't remove resouce of event */ - if (flg > 1) - evt->flags ^= evt->flags; break; #ifdef AIO_SUPPORT case EVFILT_AIO: - flg = 0; TAILQ_FOREACH_SAFE(task, &r->root_aio, task_node, tmp) { acb = (struct aiocb*) TASK_VAL(task); - if (acb != ((struct aiocb*) res[i].udata)) - continue; - else { + if (acb == ((struct aiocb*) res[i].udata)) { + if (!flg) { + TASK_RET(task) = res[i].data; + TASK_FLAG(task) = (u_long) res[i].fflags; + + /* remove user handle */ + transit_task2ready(task, &r->root_aio); + + fd = acb->aio_fildes; + if ((len = aio_return(acb)) != -1) { + if (lseek(fd, acb->aio_offset + len, SEEK_CUR) == -1) + LOGERR; + } else + LOGERR; + e_free(acb); + TASK_DATLEN(task) = (u_long) len; + TASK_FD(task) = fd; + } flg++; - TASK_RET(task) = res[i].data; - TASK_FLAG(task) = (u_long) res[i].fflags; } - /* remove user handle */ -#ifdef HAVE_LIBPTHREAD - pthread_mutex_lock(&r->root_mtx[taskAIO]); -#endif - TAILQ_REMOVE(&r->root_aio, task, task_node); -#ifdef HAVE_LIBPTHREAD - pthread_mutex_unlock(&r->root_mtx[taskAIO]); -#endif - task->task_type = taskREADY; -#ifdef HAVE_LIBPTHREAD - pthread_mutex_lock(&r->root_mtx[taskREADY]); -#endif - TAILQ_INSERT_TAIL(&r->root_ready, task, task_node); -#ifdef HAVE_LIBPTHREAD - pthread_mutex_unlock(&r->root_mtx[taskREADY]); -#endif - fd = acb->aio_fildes; - if ((len = aio_return(acb)) != -1) { - if (lseek(fd, acb->aio_offset + len, SEEK_CUR) == -1) - LOGERR; - } else - LOGERR; - free(acb); - TASK_DATLEN(task) = (u_long) len; - TASK_FD(task) = fd; } - /* if match at least 2, don't remove resouce of event */ - if (flg > 1) - evt->flags ^= evt->flags; break; #ifdef EVFILT_LIO case EVFILT_LIO: - flg = 0; TAILQ_FOREACH_SAFE(task, &r->root_lio, task_node, tmp) { acbs = (struct aiocb**) TASK_VAL(task); - if (acbs != ((struct aiocb**) res[i].udata)) - continue; - else { + if (acbs == ((struct aiocb**) res[i].udata)) { + if (!flg) { + TASK_RET(task) = res[i].data; + TASK_FLAG(task) = (u_long) res[i].fflags; + + /* remove user handle */ + transit_task2ready(task, &r->root_lio); + + iv = (struct iovec*) TASK_DATA(task); + fd = acbs[0]->aio_fildes; + off = acbs[0]->aio_offset; + for (len = 0; i < TASK_DATLEN(task); len += l, i++) { + if ((iv[i].iov_len = aio_return(acbs[i])) == -1) + l = 0; + else + l = iv[i].iov_len; + e_free(acbs[i]); + } + e_free(acbs); + TASK_DATLEN(task) = (u_long) len; + TASK_FD(task) = fd; + + if (lseek(fd, off + len, SEEK_CUR) == -1) + LOGERR; + } flg++; - TASK_RET(task) = res[i].data; - TASK_FLAG(task) = (u_long) res[i].fflags; } - /* remove user handle */ -#ifdef HAVE_LIBPTHREAD - pthread_mutex_lock(&r->root_mtx[taskLIO]); -#endif - TAILQ_REMOVE(&r->root_lio, task, task_node); -#ifdef HAVE_LIBPTHREAD - pthread_mutex_unlock(&r->root_mtx[taskLIO]); -#endif - task->task_type = taskREADY; -#ifdef HAVE_LIBPTHREAD - pthread_mutex_lock(&r->root_mtx[taskREADY]); -#endif - TAILQ_INSERT_TAIL(&r->root_ready, task, task_node); -#ifdef HAVE_LIBPTHREAD - pthread_mutex_unlock(&r->root_mtx[taskREADY]); -#endif - iv = (struct iovec*) TASK_DATA(task); - fd = acbs[0]->aio_fildes; - off = acbs[0]->aio_offset; - for (j = len = 0; i < TASK_DATLEN(task); len += l, i++) { - if ((iv[i].iov_len = aio_return(acbs[i])) == -1) - l = 0; - else - l = iv[i].iov_len; - free(acbs[i]); - } - free(acbs); - TASK_DATLEN(task) = (u_long) len; - TASK_FD(task) = fd; - - if (lseek(fd, off + len, SEEK_CUR) == -1) - LOGERR; } - /* if match at least 2, don't remove resouce of event */ - if (flg > 1) - evt->flags ^= evt->flags; break; #endif /* EVFILT_LIO */ #endif /* AIO_SUPPORT */ #ifdef EVFILT_USER case EVFILT_USER: - flg = 0; TAILQ_FOREACH_SAFE(task, &r->root_user, task_node, tmp) { - if (TASK_VAL(task) != ((uintptr_t) res[i].udata)) - continue; - else { + if (TASK_VAL(task) == ((uintptr_t) res[i].udata)) { + if (!flg) { + TASK_RET(task) = res[i].data; + TASK_FLAG(task) = (u_long) res[i].fflags; + + /* remove user handle */ + transit_task2ready(task, &r->root_user); + } flg++; - TASK_RET(task) = res[i].data; - TASK_FLAG(task) = (u_long) res[i].fflags; } - /* remove user handle */ -#ifdef HAVE_LIBPTHREAD - pthread_mutex_lock(&r->root_mtx[taskUSER]); -#endif - TAILQ_REMOVE(&r->root_user, task, task_node); -#ifdef HAVE_LIBPTHREAD - pthread_mutex_unlock(&r->root_mtx[taskUSER]); -#endif - task->task_type = taskREADY; -#ifdef HAVE_LIBPTHREAD - pthread_mutex_lock(&r->root_mtx[taskREADY]); -#endif - TAILQ_INSERT_TAIL(&r->root_ready, task, task_node); -#ifdef HAVE_LIBPTHREAD - pthread_mutex_unlock(&r->root_mtx[taskREADY]); -#endif } - /* if match at least 2, don't remove resouce of event */ - if (flg > 1) - evt->flags ^= evt->flags; break; #endif /* EVFILT_USER */ } + + if (flg > 1) + evt->flags &= ~EV_DELETE; + if (kevent(r->root_kq, evt, 1, NULL, 0, &now) == -1) { + if (r->root_hooks.hook_exec.exception) + r->root_hooks.hook_exec.exception(r, NULL); + else + LOGERR; + } + } +} +#endif + +#if SUP_ENABLE == EP_SUPPORT +static inline void +fetch_hook_epoll_proceed(int en, struct epoll_event *res, sched_root_task_t *r) +{ + register int i, rflg, wflg; + int ops = EPOLL_CTL_DEL; + sched_task_t *t, *tmp, *task; + struct epoll_event evt[1]; + + for (i = 0; i < en; i++) { + memcpy(evt, &res[i], sizeof evt); + evt->events ^= evt->events; + rflg = wflg = 0; + + if (res[i].events & (EPOLLIN | EPOLLPRI)) { + task = NULL; + TAILQ_FOREACH_SAFE(t, &r->root_read, task_node, tmp) { + if (TASK_FD(t) == evt->data.fd) { + if (!task) + task = t; + rflg++; + } + } + + if (task) { + TASK_FLAG(task) = ioctl(TASK_FD(task), FIONREAD, &TASK_RET(task)); + /* remove read handle */ + remove_task_from(task, &r->root_read); + + if (r->root_hooks.hook_exec.exception && evt->events & (EPOLLRDHUP | EPOLLERR | EPOLLHUP)) { + if (r->root_hooks.hook_exec.exception(r, (void*) (intptr_t) + (evt->events & EPOLLERR ? EV_ERROR | EV_EOF : EV_EOF))) { + task->task_type = taskUNUSE; + insert_task_to(task, &r->root_unuse); + } else { + task->task_type = taskREADY; + insert_task_to(task, &r->root_ready); + } + } else { + task->task_type = taskREADY; + insert_task_to(task, &r->root_ready); + } + + if (!(res[i].events & EPOLLOUT) && FD_ISSET(evt->data.fd, &r->root_fds[1])) { + evt->events |= EPOLLOUT; + wflg = 42; + } + if (rflg > 1) + evt->events |= EPOLLIN | EPOLLPRI; + else + FD_CLR(evt->data.fd, &r->root_fds[0]); + } + } + if (res[i].events & EPOLLOUT) { + task = NULL; + TAILQ_FOREACH_SAFE(t, &r->root_write, task_node, tmp) { + if (TASK_FD(t) == evt->data.fd) { + if (!task) + task = t; + wflg++; + } + } + + if (task) { + TASK_FLAG(task) = ioctl(TASK_FD(task), FIONWRITE, &TASK_RET(task)); + /* remove write handle */ + remove_task_from(task, &r->root_write); + + if (r->root_hooks.hook_exec.exception && evt->events & (EPOLLERR | EPOLLHUP)) { + if (r->root_hooks.hook_exec.exception(r, (void*) (intptr_t) + (evt->events & EPOLLERR ? EV_ERROR | EV_EOF : EV_EOF))) { + task->task_type = taskUNUSE; + insert_task_to(task, &r->root_unuse); + } else { + task->task_type = taskREADY; + insert_task_to(task, &r->root_ready); + } + } else { + task->task_type = taskREADY; + insert_task_to(task, &r->root_ready); + } + + if (!(res[i].events & (EPOLLIN | EPOLLPRI)) && FD_ISSET(evt->data.fd, &r->root_fds[0])) { + evt->events |= EPOLLIN | EPOLLPRI; + rflg = 42; + } + if (wflg > 1) + evt->events |= EPOLLOUT; + else + FD_CLR(evt->data.fd, &r->root_fds[1]); + } + } + + if (rflg > 1 || wflg > 1) + ops = EPOLL_CTL_MOD; + + if (epoll_ctl(r->root_kq, ops, evt->data.fd, evt) == -1) { if (r->root_hooks.hook_exec.exception) { - if (r->root_hooks.hook_exec.exception(r, NULL)) - return NULL; + r->root_hooks.hook_exec.exception(r, NULL); } else LOGERR; } } -#else /* end of kevent dispatcher */ +} +#endif + +#if SUP_ENABLE == NO_SUPPORT +static inline void +fetch_hook_select_proceed(int en, fd_set rfd, fd_set wfd, fd_set xfd, sched_root_task_t *r) +{ + register int i, rflg, wflg; + sched_task_t *t, *tmp, *task; + + /* skip select check if return value from select is zero */ + if (!en) + return; + for (i = 0; i < r->root_kq; i++) { + if (!FD_ISSET(i, &r->root_fds[0]) && !FD_ISSET(i, &r->root_fds[1])) + continue; + + rflg = wflg = 0; + if (FD_ISSET(i, &rfd) || FD_ISSET(i, &xfd)) { - flg = 0; - TAILQ_FOREACH_SAFE(task, &r->root_read, task_node, tmp) { - if (TASK_FD(task) != i) - continue; - else { - flg++; - TASK_FLAG(task) = ioctl(TASK_FD(task), - FIONREAD, &TASK_RET(task)); + task = NULL; + TAILQ_FOREACH_SAFE(t, &r->root_read, task_node, tmp) { + if (TASK_FD(t) == i) { + if (!task) + task = t; + rflg++; } + } + + if (task) { + TASK_FLAG(task) = ioctl(TASK_FD(task), FIONREAD, &TASK_RET(task)); + /* remove read handle */ -#ifdef HAVE_LIBPTHREAD - pthread_mutex_lock(&r->root_mtx[taskREAD]); -#endif - TAILQ_REMOVE(&r->root_read, task, task_node); -#ifdef HAVE_LIBPTHREAD - pthread_mutex_unlock(&r->root_mtx[taskREAD]); -#endif + remove_task_from(task, &r->root_read); + if (r->root_hooks.hook_exec.exception) { if (r->root_hooks.hook_exec.exception(r, NULL)) { task->task_type = taskUNUSE; -#ifdef HAVE_LIBPTHREAD - pthread_mutex_lock(&r->root_mtx[taskUNUSE]); -#endif - TAILQ_INSERT_TAIL(&r->root_unuse, task, task_node); -#ifdef HAVE_LIBPTHREAD - pthread_mutex_unlock(&r->root_mtx[taskUNUSE]); -#endif + insert_task_to(task, &r->root_unuse); } else { task->task_type = taskREADY; -#ifdef HAVE_LIBPTHREAD - pthread_mutex_lock(&r->root_mtx[taskREADY]); -#endif - TAILQ_INSERT_TAIL(&r->root_ready, task, task_node); -#ifdef HAVE_LIBPTHREAD - pthread_mutex_unlock(&r->root_mtx[taskREADY]); -#endif + insert_task_to(task, &r->root_ready); } } else { task->task_type = taskREADY; -#ifdef HAVE_LIBPTHREAD - pthread_mutex_lock(&r->root_mtx[taskREADY]); -#endif - TAILQ_INSERT_TAIL(&r->root_ready, task, task_node); -#ifdef HAVE_LIBPTHREAD - pthread_mutex_unlock(&r->root_mtx[taskREADY]); -#endif + insert_task_to(task, &r->root_ready); } + + /* remove resouce */ + if (rflg == 1) + FD_CLR(i, &r->root_fds[0]); } - /* if match equal to 1, remove resouce */ - if (flg == 1) - FD_CLR(i, &r->root_fds[0]); } - if (FD_ISSET(i, &wfd)) { - flg = 0; - TAILQ_FOREACH_SAFE(task, &r->root_write, task_node, tmp) { - if (TASK_FD(task) != i) - continue; - else { - flg++; - TASK_FLAG(task) = ioctl(TASK_FD(task), - FIONWRITE, &TASK_RET(task)); + task = NULL; + TAILQ_FOREACH_SAFE(t, &r->root_write, task_node, tmp) { + if (TASK_FD(t) == i) { + if (!task) + task = t; + wflg++; } + } + + if (task) { + TASK_FLAG(task) = ioctl(TASK_FD(task), FIONWRITE, &TASK_RET(task)); + /* remove write handle */ -#ifdef HAVE_LIBPTHREAD - pthread_mutex_lock(&r->root_mtx[taskWRITE]); -#endif - TAILQ_REMOVE(&r->root_write, task, task_node); -#ifdef HAVE_LIBPTHREAD - pthread_mutex_unlock(&r->root_mtx[taskWRITE]); -#endif + remove_task_from(task, &r->root_write); + if (r->root_hooks.hook_exec.exception) { if (r->root_hooks.hook_exec.exception(r, NULL)) { task->task_type = taskUNUSE; -#ifdef HAVE_LIBPTHREAD - pthread_mutex_lock(&r->root_mtx[taskUNUSE]); -#endif - TAILQ_INSERT_TAIL(&r->root_unuse, task, task_node); -#ifdef HAVE_LIBPTHREAD - pthread_mutex_unlock(&r->root_mtx[taskUNUSE]); -#endif + insert_task_to(task, &r->root_unuse); } else { task->task_type = taskREADY; -#ifdef HAVE_LIBPTHREAD - pthread_mutex_lock(&r->root_mtx[taskREADY]); -#endif - TAILQ_INSERT_TAIL(&r->root_ready, task, task_node); -#ifdef HAVE_LIBPTHREAD - pthread_mutex_unlock(&r->root_mtx[taskREADY]); -#endif + insert_task_to(task, &r->root_ready); } } else { task->task_type = taskREADY; -#ifdef HAVE_LIBPTHREAD - pthread_mutex_lock(&r->root_mtx[taskREADY]); -#endif - TAILQ_INSERT_TAIL(&r->root_ready, task, task_node); -#ifdef HAVE_LIBPTHREAD - pthread_mutex_unlock(&r->root_mtx[taskREADY]); -#endif + insert_task_to(task, &r->root_ready); } + + /* remove resouce */ + if (wflg == 1) + FD_CLR(i, &r->root_fds[1]); } - /* if match equal to 1, remove resouce */ - if (flg == 1) - FD_CLR(i, &r->root_fds[1]); } } /* optimize select */ - for (i = r->root_kq - 1; i > 2; i--) + for (i = r->root_kq - 1; i >= 0; i--) if (FD_ISSET(i, &r->root_fds[0]) || FD_ISSET(i, &r->root_fds[1])) break; - if (i > 2) - r->root_kq = i + 1; -#endif /* KQ_DISABLE */ + r->root_kq = i + 1; +} +#endif +/* + * sched_hook_fetch() - Default FETCH hook + * + * @root = root task + * @arg = unused + * return: NULL error or !=NULL fetched task + */ +void * +sched_hook_fetch(void *root, void *arg __unused) +{ + sched_root_task_t *r = root; + sched_task_t *task, *tmp; + struct timespec now, m, mtmp; +#if SUP_ENABLE == KQ_SUPPORT + struct kevent res[KQ_EVENTS]; + struct timespec *timeout; +#elif SUP_ENABLE == EP_SUPPORT + struct epoll_event res[KQ_EVENTS]; + u_long timeout = 0; +#else + struct timeval *timeout, tv; + fd_set rfd, wfd, xfd; +#endif + int en; + + if (!r) + return NULL; + + /* get new task by queue priority */ + while ((task = TAILQ_FIRST(&r->root_event))) { + transit_task2unuse(task, &r->root_event); + return task; + } + while ((task = TAILQ_FIRST(&r->root_ready))) { + transit_task2unuse(task, &r->root_ready); + return task; + } + + /* if present member of task, set NOWAIT */ + if (!TAILQ_FIRST(&r->root_task)) { + /* timer tasks */ +#ifdef TIMER_WITHOUT_SORT + clock_gettime(CLOCK_MONOTONIC, &now); + + sched_timespecclear(&r->root_wait); + TAILQ_FOREACH(task, &r->root_timer, task_node) { + if (!sched_timespecisset(&r->root_wait)) + r->root_wait = TASK_TS(task); + else if (sched_timespeccmp(&TASK_TS(task), &r->root_wait, -) < 0) + r->root_wait = TASK_TS(task); + } + + if (TAILQ_FIRST(&r->root_timer)) { + m = r->root_wait; + sched_timespecsub(&m, &now, &mtmp); + r->root_wait = mtmp; + } else { + /* set wait INFTIM */ + sched_timespecinf(&r->root_wait); + } +#else /* ! TIMER_WITHOUT_SORT */ + if ((task = TAILQ_FIRST(&r->root_timer))) { + clock_gettime(CLOCK_MONOTONIC, &now); + + m = TASK_TS(task); + sched_timespecsub(&m, &now, &mtmp); + r->root_wait = mtmp; + } else { + /* set wait INFTIM */ + sched_timespecinf(&r->root_wait); + } +#endif /* TIMER_WITHOUT_SORT */ + } else /* no waiting for event, because we have ready task */ + sched_timespecclear(&r->root_wait); + + if (r->root_wait.tv_sec != -1 && r->root_wait.tv_nsec != -1) { +#if SUP_ENABLE == KQ_SUPPORT + timeout = &r->root_wait; +#elif SUP_ENABLE == EP_SUPPORT + timeout = r->root_wait.tv_sec * 1000 + r->root_wait.tv_nsec / 1000000; +#else + sched_timespec2val(&r->root_wait, &tv); + timeout = &tv; +#endif /* KQ_SUPPORT */ + } else if (sched_timespecisinf(&r->root_poll)) +#if SUP_ENABLE == EP_SUPPORT + timeout = -1; +#else + timeout = NULL; +#endif + else { +#if SUP_ENABLE == KQ_SUPPORT + timeout = &r->root_poll; +#elif SUP_ENABLE == EP_SUPPORT + timeout = r->root_poll.tv_sec * 1000 + r->root_poll.tv_nsec / 1000000; +#else + sched_timespec2val(&r->root_poll, &tv); + timeout = &tv; +#endif /* KQ_SUPPORT */ + } + +#if SUP_ENABLE == KQ_SUPPORT + if ((en = kevent(r->root_kq, NULL, 0, res, KQ_EVENTS, timeout)) == -1) { +#elif SUP_ENABLE == EP_SUPPORT + if ((en = epoll_wait(r->root_kq, res, KQ_EVENTS, timeout)) == -1) { +#else + rfd = xfd = r->root_fds[0]; + wfd = r->root_fds[1]; + if ((en = select(r->root_kq, &rfd, &wfd, &xfd, timeout)) == -1) { +#endif /* KQ_SUPPORT */ + if (r->root_hooks.hook_exec.exception) { + if (r->root_hooks.hook_exec.exception(r, NULL)) + return NULL; + } else if (errno != EINTR) + LOGERR; + goto skip_event; + } + + /* Go and catch the cat into pipes ... */ +#if SUP_ENABLE == KQ_SUPPORT + /* kevent dispatcher */ + fetch_hook_kevent_proceed(en, res, r); +#elif SUP_ENABLE == EP_SUPPORT + /* epoll dispatcher */ + fetch_hook_epoll_proceed(en, res, r); +#else + /* select dispatcher */ + fetch_hook_select_proceed(en, rfd, wfd, xfd, r); +#endif /* KQ_SUPPORT */ + skip_event: /* timer update & put in ready queue */ clock_gettime(CLOCK_MONOTONIC, &now); TAILQ_FOREACH_SAFE(task, &r->root_timer, task_node, tmp) - if (sched_timespeccmp(&now, &TASK_TS(task), -) >= 0) { -#ifdef HAVE_LIBPTHREAD - pthread_mutex_lock(&r->root_mtx[taskTIMER]); -#endif - TAILQ_REMOVE(&r->root_timer, task, task_node); -#ifdef HAVE_LIBPTHREAD - pthread_mutex_unlock(&r->root_mtx[taskTIMER]); -#endif - task->task_type = taskREADY; -#ifdef HAVE_LIBPTHREAD - pthread_mutex_lock(&r->root_mtx[taskREADY]); -#endif - TAILQ_INSERT_TAIL(&r->root_ready, task, task_node); -#ifdef HAVE_LIBPTHREAD - pthread_mutex_unlock(&r->root_mtx[taskREADY]); -#endif - } + if (sched_timespeccmp(&now, &TASK_TS(task), -) >= 0) + transit_task2ready(task, &r->root_timer); /* put regular task priority task to ready queue, if there is no ready task or reach max missing hit for regular task */ @@ -1288,21 +1405,7 @@ skip_event: if (!TAILQ_FIRST(&r->root_ready) || r->root_miss >= TASK_VAL(task)) { r->root_miss ^= r->root_miss; -#ifdef HAVE_LIBPTHREAD - pthread_mutex_lock(&r->root_mtx[taskTASK]); -#endif - TAILQ_REMOVE(&r->root_task, task, task_node); -#ifdef HAVE_LIBPTHREAD - pthread_mutex_unlock(&r->root_mtx[taskTASK]); -#endif - task->task_type = taskREADY; -#ifdef HAVE_LIBPTHREAD - pthread_mutex_lock(&r->root_mtx[taskREADY]); -#endif - TAILQ_INSERT_TAIL(&r->root_ready, task, task_node); -#ifdef HAVE_LIBPTHREAD - pthread_mutex_unlock(&r->root_mtx[taskREADY]); -#endif + transit_task2ready(task, &r->root_task); } else r->root_miss++; } else @@ -1310,24 +1413,8 @@ skip_event: /* OK, lets get ready task !!! */ task = TAILQ_FIRST(&r->root_ready); - if (!(task)) - return NULL; - -#ifdef HAVE_LIBPTHREAD - pthread_mutex_lock(&r->root_mtx[taskREADY]); -#endif - TAILQ_REMOVE(&r->root_ready, task, task_node); -#ifdef HAVE_LIBPTHREAD - pthread_mutex_unlock(&r->root_mtx[taskREADY]); -#endif - task->task_type = taskUNUSE; -#ifdef HAVE_LIBPTHREAD - pthread_mutex_lock(&r->root_mtx[taskUNUSE]); -#endif - TAILQ_INSERT_TAIL(&r->root_unuse, task, task_node); -#ifdef HAVE_LIBPTHREAD - pthread_mutex_unlock(&r->root_mtx[taskUNUSE]); -#endif + if (task) + transit_task2unuse(task, &r->root_ready); return task; } @@ -1377,7 +1464,7 @@ sched_hook_condition(void *root, void *arg) if (!r) return NULL; - return (void*) (r->root_cond - *(intptr_t*) arg); + return (void*) (*r->root_cond - *(intptr_t*) arg); } /* @@ -1387,14 +1474,18 @@ sched_hook_condition(void *root, void *arg) * @arg = unused * return: <0 errors and 0 ok */ -#if defined(HAVE_TIMER_CREATE) && defined(HAVE_TIMER_SETTIME) && defined(HAVE_TIMER_DELETE) void * sched_hook_rtc(void *task, void *arg __unused) { +#if defined(HAVE_LIBRT) && defined(HAVE_TIMER_CREATE) && \ + defined(HAVE_TIMER_SETTIME) && defined(HAVE_TIMER_DELETE) sched_task_t *sigt = NULL, *t = task; struct itimerspec its; struct sigevent evt; timer_t tmr; +#if SUP_ENABLE != KQ_SUPPORT + struct sigaction sa; +#endif if (!t || !TASK_ROOT(t)) return (void*) -1; @@ -1402,7 +1493,7 @@ sched_hook_rtc(void *task, void *arg __unused) memset(&evt, 0, sizeof evt); evt.sigev_notify = SIGEV_SIGNAL; evt.sigev_signo = (intptr_t) TASK_DATA(t) + SIGRTMIN; - evt.sigev_value.sival_ptr = TASK_DATA(t); + evt.sigev_value.sival_ptr = t; if (timer_create(CLOCK_MONOTONIC, &evt, &tmr) == -1) { if (TASK_ROOT(t)->root_hooks.hook_exec.exception) @@ -1413,6 +1504,7 @@ sched_hook_rtc(void *task, void *arg __unused) } else TASK_FLAG(t) = (u_long) tmr; +#if SUP_ENABLE == KQ_SUPPORT if (!(sigt = schedSignal(TASK_ROOT(t), _sched_rtcWrapper, TASK_ARG(t), evt.sigev_signo, t, (size_t) tmr))) { if (TASK_ROOT(t)->root_hooks.hook_exec.exception) @@ -1423,7 +1515,22 @@ sched_hook_rtc(void *task, void *arg __unused) return (void*) -1; } else TASK_RET(t) = (uintptr_t) sigt; +#else + memset(&sa, 0, sizeof sa); + sigemptyset(&sa.sa_mask); + sa.sa_sigaction = _sched_rtcSigWrapper; + sa.sa_flags = SA_SIGINFO | SA_RESTART; + if (sigaction(evt.sigev_signo, &sa, NULL) == -1) { + if (TASK_ROOT(t)->root_hooks.hook_exec.exception) + TASK_ROOT(t)->root_hooks.hook_exec.exception(TASK_ROOT(t), NULL); + else + LOGERR; + timer_delete(tmr); + return (void*) -1; + } +#endif + memset(&its, 0, sizeof its); its.it_value.tv_sec = t->task_val.ts.tv_sec; its.it_value.tv_nsec = t->task_val.ts.tv_nsec; @@ -1437,7 +1544,6 @@ sched_hook_rtc(void *task, void *arg __unused) timer_delete(tmr); return (void*) -1; } - +#endif /* HAVE_TIMER_CREATE */ return NULL; } -#endif /* HAVE_TIMER_CREATE */