![[BACK]](/icons/cvsweb/back.gif){kind=icon}
Return to stat_cache.c CVS log ![[TXT]](/icons/cvsweb/text.gif){kind=icon}
![[DIR]](/icons/cvsweb/dir.gif){kind=icon}
Up to [ELWIX - Embedded LightWeight unIX -] / embedaddon / lighttpd / src|
Return to stat_cache.c CVS log | Up to [ELWIX - Embedded LightWeight unIX -] / embedaddon / lighttpd / src |
1.1 misho 1: #include "log.h"
2: #include "stat_cache.h"
3: #include "fdevent.h"
4: #include "etag.h"
5:
6: #include <sys/types.h>
7: #include <sys/stat.h>
8:
9: #include <stdlib.h>
10: #include <string.h>
11: #include <errno.h>
12: #include <unistd.h>
13: #include <stdio.h>
14: #include <fcntl.h>
15: #include <assert.h>
16:
17: #ifdef HAVE_ATTR_ATTRIBUTES_H
18: # include <attr/attributes.h>
19: #endif
20:
21: #ifdef HAVE_FAM_H
22: # include <fam.h>
23: #endif
24:
25: #include "sys-mmap.h"
26:
27: /* NetBSD 1.3.x needs it */
28: #ifndef MAP_FAILED
29: # define MAP_FAILED -1
30: #endif
31:
32: #ifndef O_LARGEFILE
33: # define O_LARGEFILE 0
34: #endif
35:
36: #ifndef HAVE_LSTAT
37: # define lstat stat
38: #endif
39:
40: #if 0
41: /* enables debug code for testing if all nodes in the stat-cache as accessable */
42: #define DEBUG_STAT_CACHE
43: #endif
44:
45: /*
46: * stat-cache
47: *
48: * we cache the stat() calls in our own storage
49: * the directories are cached in FAM
50: *
51: * if we get a change-event from FAM, we increment the version in the FAM->dir mapping
52: *
53: * if the stat()-cache is queried we check if the version id for the directory is the
54: * same and return immediatly.
55: *
56: *
57: * What we need:
58: *
59: * - for each stat-cache entry we need a fast indirect lookup on the directory name
60: * - for each FAMRequest we have to find the version in the directory cache (index as userdata)
61: *
62: * stat <<-> directory <-> FAMRequest
63: *
64: * if file is deleted, directory is dirty, file is rechecked ...
65: * if directory is deleted, directory mapping is removed
66: *
67: * */
68:
69: #ifdef HAVE_FAM_H
70: typedef struct {
71: FAMRequest *req;
72:
73: buffer *name;
74:
75: int version;
76: } fam_dir_entry;
77: #endif
78:
79: /* the directory name is too long to always compare on it
80: * - we need a hash
81: * - the hash-key is used as sorting criteria for a tree
82: * - a splay-tree is used as we can use the caching effect of it
83: */
84:
85: /* we want to cleanup the stat-cache every few seconds, let's say 10
86: *
87: * - remove entries which are outdated since 30s
88: * - remove entries which are fresh but havn't been used since 60s
89: * - if we don't have a stat-cache entry for a directory, release it from the monitor
90: */
91:
92: #ifdef DEBUG_STAT_CACHE
93: typedef struct {
94: int *ptr;
95:
96: size_t used;
97: size_t size;
98: } fake_keys;
99:
100: static fake_keys ctrl;
101: #endif
102:
103: stat_cache *stat_cache_init(void) {
1.1.1.2 ! misho 104: stat_cache *sc = NULL;
1.1 misho 105:
1.1.1.2 ! misho 106: sc = calloc(1, sizeof(*sc));
! 107:
! 108: sc->dir_name = buffer_init();
! 109: sc->hash_key = buffer_init();
1.1 misho 110:
111: #ifdef HAVE_FAM_H
1.1.1.2 ! misho 112: sc->fam_fcce_ndx = -1;
1.1 misho 113: #endif
114:
115: #ifdef DEBUG_STAT_CACHE
116: ctrl.size = 0;
117: #endif
118:
1.1.1.2 ! misho 119: return sc;
1.1 misho 120: }
121:
122: static stat_cache_entry * stat_cache_entry_init(void) {
123: stat_cache_entry *sce = NULL;
124:
125: sce = calloc(1, sizeof(*sce));
126:
127: sce->name = buffer_init();
128: sce->etag = buffer_init();
129: sce->content_type = buffer_init();
130:
131: return sce;
132: }
133:
134: static void stat_cache_entry_free(void *data) {
135: stat_cache_entry *sce = data;
136: if (!sce) return;
137:
138: buffer_free(sce->etag);
139: buffer_free(sce->name);
140: buffer_free(sce->content_type);
141:
142: free(sce);
143: }
144:
145: #ifdef HAVE_FAM_H
146: static fam_dir_entry * fam_dir_entry_init(void) {
147: fam_dir_entry *fam_dir = NULL;
148:
149: fam_dir = calloc(1, sizeof(*fam_dir));
150:
151: fam_dir->name = buffer_init();
152:
153: return fam_dir;
154: }
155:
1.1.1.2 ! misho 156: static void fam_dir_entry_free(FAMConnection *fc, void *data) {
1.1 misho 157: fam_dir_entry *fam_dir = data;
158:
159: if (!fam_dir) return;
160:
1.1.1.2 ! misho 161: FAMCancelMonitor(fc, fam_dir->req);
1.1 misho 162:
163: buffer_free(fam_dir->name);
164: free(fam_dir->req);
165:
166: free(fam_dir);
167: }
168: #endif
169:
170: void stat_cache_free(stat_cache *sc) {
171: while (sc->files) {
172: int osize;
173: splay_tree *node = sc->files;
174:
175: osize = sc->files->size;
176:
177: stat_cache_entry_free(node->data);
178: sc->files = splaytree_delete(sc->files, node->key);
179:
1.1.1.2 ! misho 180: force_assert(osize - 1 == splaytree_size(sc->files));
1.1 misho 181: }
182:
183: buffer_free(sc->dir_name);
184: buffer_free(sc->hash_key);
185:
186: #ifdef HAVE_FAM_H
187: while (sc->dirs) {
188: int osize;
189: splay_tree *node = sc->dirs;
190:
191: osize = sc->dirs->size;
192:
1.1.1.2 ! misho 193: fam_dir_entry_free(&sc->fam, node->data);
1.1 misho 194: sc->dirs = splaytree_delete(sc->dirs, node->key);
195:
196: if (osize == 1) {
1.1.1.2 ! misho 197: force_assert(NULL == sc->dirs);
1.1 misho 198: } else {
1.1.1.2 ! misho 199: force_assert(osize == (sc->dirs->size + 1));
1.1 misho 200: }
201: }
202:
1.1.1.2 ! misho 203: if (-1 != sc->fam_fcce_ndx) {
! 204: /* fd events already gone */
! 205: sc->fam_fcce_ndx = -1;
! 206:
! 207: FAMClose(&sc->fam);
1.1 misho 208: }
209: #endif
210: free(sc);
211: }
212:
213: #ifdef HAVE_XATTR
214: static int stat_cache_attr_get(buffer *buf, char *name) {
215: int attrlen;
216: int ret;
217:
218: attrlen = 1024;
219: buffer_prepare_copy(buf, attrlen);
220: attrlen--;
221: if(0 == (ret = attr_get(name, "Content-Type", buf->ptr, &attrlen, 0))) {
222: buf->used = attrlen + 1;
223: buf->ptr[attrlen] = '\0';
224: }
225: return ret;
226: }
227: #endif
228:
229: /* the famous DJB hash function for strings */
230: static uint32_t hashme(buffer *str) {
231: uint32_t hash = 5381;
232: const char *s;
233: for (s = str->ptr; *s; s++) {
234: hash = ((hash << 5) + hash) + *s;
235: }
236:
237: hash &= ~(1 << 31); /* strip the highest bit */
238:
239: return hash;
240: }
241:
242: #ifdef HAVE_FAM_H
243: handler_t stat_cache_handle_fdevent(server *srv, void *_fce, int revent) {
244: size_t i;
245: stat_cache *sc = srv->stat_cache;
246: size_t events;
247:
248: UNUSED(_fce);
249: /* */
250:
1.1.1.2 ! misho 251: if (revent & FDEVENT_IN) {
! 252: events = FAMPending(&sc->fam);
1.1 misho 253:
254: for (i = 0; i < events; i++) {
255: FAMEvent fe;
256: fam_dir_entry *fam_dir;
257: splay_tree *node;
258: int ndx, j;
259:
1.1.1.2 ! misho 260: FAMNextEvent(&sc->fam, &fe);
1.1 misho 261:
262: /* handle event */
263:
264: switch(fe.code) {
265: case FAMChanged:
266: case FAMDeleted:
267: case FAMMoved:
268: /* if the filename is a directory remove the entry */
269:
270: fam_dir = fe.userdata;
271: fam_dir->version++;
272:
273: /* file/dir is still here */
274: if (fe.code == FAMChanged) break;
275:
276: /* we have 2 versions, follow and no-follow-symlink */
277:
278: for (j = 0; j < 2; j++) {
279: buffer_copy_string(sc->hash_key, fe.filename);
280: buffer_append_long(sc->hash_key, j);
281:
282: ndx = hashme(sc->hash_key);
283:
284: sc->dirs = splaytree_splay(sc->dirs, ndx);
285: node = sc->dirs;
286:
287: if (node && (node->key == ndx)) {
288: int osize = splaytree_size(sc->dirs);
289:
1.1.1.2 ! misho 290: fam_dir_entry_free(&sc->fam, node->data);
1.1 misho 291: sc->dirs = splaytree_delete(sc->dirs, ndx);
292:
1.1.1.2 ! misho 293: force_assert(osize - 1 == splaytree_size(sc->dirs));
1.1 misho 294: }
295: }
296: break;
297: default:
298: break;
299: }
300: }
301: }
302:
303: if (revent & FDEVENT_HUP) {
304: /* fam closed the connection */
1.1.1.2 ! misho 305: fdevent_event_del(srv->ev, &(sc->fam_fcce_ndx), FAMCONNECTION_GETFD(&sc->fam));
! 306: fdevent_unregister(srv->ev, FAMCONNECTION_GETFD(&sc->fam));
1.1 misho 307:
1.1.1.2 ! misho 308: FAMClose(&sc->fam);
1.1 misho 309: }
310:
311: return HANDLER_GO_ON;
312: }
313:
314: static int buffer_copy_dirname(buffer *dst, buffer *file) {
315: size_t i;
316:
317: if (buffer_is_empty(file)) return -1;
318:
319: for (i = file->used - 1; i+1 > 0; i--) {
320: if (file->ptr[i] == '/') {
321: buffer_copy_string_len(dst, file->ptr, i);
322: return 0;
323: }
324: }
325:
326: return -1;
327: }
328: #endif
329:
330: #ifdef HAVE_LSTAT
331: static int stat_cache_lstat(server *srv, buffer *dname, struct stat *lst) {
332: if (lstat(dname->ptr, lst) == 0) {
333: return S_ISLNK(lst->st_mode) ? 0 : 1;
334: }
335: else {
336: log_error_write(srv, __FILE__, __LINE__, "sbs",
337: "lstat failed for:",
338: dname, strerror(errno));
339: };
340: return -1;
341: }
342: #endif
343:
344: /***
345: *
346: *
347: *
348: * returns:
349: * - HANDLER_FINISHED on cache-miss (don't forget to reopen the file)
350: * - HANDLER_ERROR on stat() failed -> see errno for problem
351: */
352:
353: handler_t stat_cache_get_entry(server *srv, connection *con, buffer *name, stat_cache_entry **ret_sce) {
354: #ifdef HAVE_FAM_H
355: fam_dir_entry *fam_dir = NULL;
356: int dir_ndx = -1;
357: splay_tree *dir_node = NULL;
358: #endif
359: stat_cache_entry *sce = NULL;
360: stat_cache *sc;
361: struct stat st;
362: size_t k;
363: int fd;
364: struct stat lst;
365: #ifdef DEBUG_STAT_CACHE
366: size_t i;
367: #endif
368:
369: int file_ndx;
370: splay_tree *file_node = NULL;
371:
372: *ret_sce = NULL;
373:
374: /*
375: * check if the directory for this file has changed
376: */
377:
378: sc = srv->stat_cache;
379:
380: buffer_copy_string_buffer(sc->hash_key, name);
381: buffer_append_long(sc->hash_key, con->conf.follow_symlink);
382:
383: file_ndx = hashme(sc->hash_key);
384: sc->files = splaytree_splay(sc->files, file_ndx);
385:
386: #ifdef DEBUG_STAT_CACHE
387: for (i = 0; i < ctrl.used; i++) {
388: if (ctrl.ptr[i] == file_ndx) break;
389: }
390: #endif
391:
392: if (sc->files && (sc->files->key == file_ndx)) {
393: #ifdef DEBUG_STAT_CACHE
394: /* it was in the cache */
1.1.1.2 ! misho 395: force_assert(i < ctrl.used);
1.1 misho 396: #endif
397:
398: /* we have seen this file already and
399: * don't stat() it again in the same second */
400:
401: file_node = sc->files;
402:
403: sce = file_node->data;
404:
405: /* check if the name is the same, we might have a collision */
406:
407: if (buffer_is_equal(name, sce->name)) {
408: if (srv->srvconf.stat_cache_engine == STAT_CACHE_ENGINE_SIMPLE) {
409: if (sce->stat_ts == srv->cur_ts) {
410: *ret_sce = sce;
411: return HANDLER_GO_ON;
412: }
413: }
414: } else {
415: /* oops, a collision,
416: *
417: * file_node is used by the FAM check below to see if we know this file
418: * and if we can save a stat().
419: *
420: * BUT, the sce is not reset here as the entry into the cache is ok, we
421: * it is just not pointing to our requested file.
422: *
423: * */
424:
425: file_node = NULL;
426: }
427: } else {
428: #ifdef DEBUG_STAT_CACHE
429: if (i != ctrl.used) {
430: log_error_write(srv, __FILE__, __LINE__, "xSB",
431: file_ndx, "was already inserted but not found in cache, ", name);
432: }
1.1.1.2 ! misho 433: force_assert(i == ctrl.used);
1.1 misho 434: #endif
435: }
436:
437: #ifdef HAVE_FAM_H
438: /* dir-check */
439: if (srv->srvconf.stat_cache_engine == STAT_CACHE_ENGINE_FAM) {
440: if (0 != buffer_copy_dirname(sc->dir_name, name)) {
441: log_error_write(srv, __FILE__, __LINE__, "sb",
442: "no '/' found in filename:", name);
443: return HANDLER_ERROR;
444: }
445:
446: buffer_copy_string_buffer(sc->hash_key, sc->dir_name);
447: buffer_append_long(sc->hash_key, con->conf.follow_symlink);
448:
449: dir_ndx = hashme(sc->hash_key);
450:
451: sc->dirs = splaytree_splay(sc->dirs, dir_ndx);
452:
453: if (sc->dirs && (sc->dirs->key == dir_ndx)) {
454: dir_node = sc->dirs;
455: }
456:
457: if (dir_node && file_node) {
458: /* we found a file */
459:
460: sce = file_node->data;
461: fam_dir = dir_node->data;
462:
463: if (fam_dir->version == sce->dir_version) {
464: /* the stat()-cache entry is still ok */
465:
466: *ret_sce = sce;
467: return HANDLER_GO_ON;
468: }
469: }
470: }
471: #endif
472:
473: /*
474: * *lol*
475: * - open() + fstat() on a named-pipe results in a (intended) hang.
476: * - stat() if regular file + open() to see if we can read from it is better
477: *
478: * */
479: if (-1 == stat(name->ptr, &st)) {
480: return HANDLER_ERROR;
481: }
482:
483:
484: if (S_ISREG(st.st_mode)) {
485: /* fix broken stat/open for symlinks to reg files with appended slash on freebsd,osx */
486: if (name->ptr[name->used-2] == '/') {
487: errno = ENOTDIR;
488: return HANDLER_ERROR;
489: }
490:
491: /* try to open the file to check if we can read it */
492: if (-1 == (fd = open(name->ptr, O_RDONLY))) {
493: return HANDLER_ERROR;
494: }
495: close(fd);
496: }
497:
498: if (NULL == sce) {
499: #ifdef DEBUG_STAT_CACHE
500: int osize = splaytree_size(sc->files);
501: #endif
502:
503: sce = stat_cache_entry_init();
504: buffer_copy_string_buffer(sce->name, name);
505:
506: sc->files = splaytree_insert(sc->files, file_ndx, sce);
507: #ifdef DEBUG_STAT_CACHE
508: if (ctrl.size == 0) {
509: ctrl.size = 16;
510: ctrl.used = 0;
511: ctrl.ptr = malloc(ctrl.size * sizeof(*ctrl.ptr));
512: } else if (ctrl.size == ctrl.used) {
513: ctrl.size += 16;
514: ctrl.ptr = realloc(ctrl.ptr, ctrl.size * sizeof(*ctrl.ptr));
515: }
516:
517: ctrl.ptr[ctrl.used++] = file_ndx;
518:
1.1.1.2 ! misho 519: force_assert(sc->files);
! 520: force_assert(sc->files->data == sce);
! 521: force_assert(osize + 1 == splaytree_size(sc->files));
1.1 misho 522: #endif
523: }
524:
525: sce->st = st;
526: sce->stat_ts = srv->cur_ts;
527:
528: /* catch the obvious symlinks
529: *
530: * this is not a secure check as we still have a race-condition between
531: * the stat() and the open. We can only solve this by
532: * 1. open() the file
533: * 2. fstat() the fd
534: *
535: * and keeping the file open for the rest of the time. But this can
536: * only be done at network level.
537: *
538: * per default it is not a symlink
539: * */
540: #ifdef HAVE_LSTAT
541: sce->is_symlink = 0;
542:
543: /* we want to only check for symlinks if we should block symlinks.
544: */
545: if (!con->conf.follow_symlink) {
546: if (stat_cache_lstat(srv, name, &lst) == 0) {
547: #ifdef DEBUG_STAT_CACHE
548: log_error_write(srv, __FILE__, __LINE__, "sb",
549: "found symlink", name);
550: #endif
551: sce->is_symlink = 1;
552: }
553:
554: /*
555: * we assume "/" can not be symlink, so
556: * skip the symlink stuff if our path is /
557: **/
558: else if ((name->used > 2)) {
559: buffer *dname;
560: char *s_cur;
561:
562: dname = buffer_init();
563: buffer_copy_string_buffer(dname, name);
564:
565: while ((s_cur = strrchr(dname->ptr,'/'))) {
566: *s_cur = '\0';
567: dname->used = s_cur - dname->ptr + 1;
568: if (dname->ptr == s_cur) {
569: #ifdef DEBUG_STAT_CACHE
570: log_error_write(srv, __FILE__, __LINE__, "s", "reached /");
571: #endif
572: break;
573: }
574: #ifdef DEBUG_STAT_CACHE
575: log_error_write(srv, __FILE__, __LINE__, "sbs",
576: "checking if", dname, "is a symlink");
577: #endif
578: if (stat_cache_lstat(srv, dname, &lst) == 0) {
579: sce->is_symlink = 1;
580: #ifdef DEBUG_STAT_CACHE
581: log_error_write(srv, __FILE__, __LINE__, "sb",
582: "found symlink", dname);
583: #endif
584: break;
585: };
586: };
587: buffer_free(dname);
588: };
589: };
590: #endif
591:
592: if (S_ISREG(st.st_mode)) {
593: /* determine mimetype */
594: buffer_reset(sce->content_type);
595: #ifdef HAVE_XATTR
596: if (con->conf.use_xattr) {
597: stat_cache_attr_get(sce->content_type, name->ptr);
598: }
599: #endif
600: /* xattr did not set a content-type. ask the config */
601: if (buffer_is_empty(sce->content_type)) {
602: for (k = 0; k < con->conf.mimetypes->used; k++) {
603: data_string *ds = (data_string *)con->conf.mimetypes->data[k];
604: buffer *type = ds->key;
605:
606: if (type->used == 0) continue;
607:
608: /* check if the right side is the same */
609: if (type->used > name->used) continue;
610:
611: if (0 == strncasecmp(name->ptr + name->used - type->used, type->ptr, type->used - 1)) {
612: buffer_copy_string_buffer(sce->content_type, ds->value);
613: break;
614: }
615: }
616: }
617: etag_create(sce->etag, &(sce->st), con->etag_flags);
618: } else if (S_ISDIR(st.st_mode)) {
619: etag_create(sce->etag, &(sce->st), con->etag_flags);
620: }
621:
622: #ifdef HAVE_FAM_H
1.1.1.2 ! misho 623: if (srv->srvconf.stat_cache_engine == STAT_CACHE_ENGINE_FAM) {
1.1 misho 624: /* is this directory already registered ? */
625: if (!dir_node) {
626: fam_dir = fam_dir_entry_init();
627:
628: buffer_copy_string_buffer(fam_dir->name, sc->dir_name);
629:
630: fam_dir->version = 1;
631:
632: fam_dir->req = calloc(1, sizeof(FAMRequest));
633:
1.1.1.2 ! misho 634: if (0 != FAMMonitorDirectory(&sc->fam, fam_dir->name->ptr,
1.1 misho 635: fam_dir->req, fam_dir)) {
636:
637: log_error_write(srv, __FILE__, __LINE__, "sbsbs",
638: "monitoring dir failed:",
639: fam_dir->name,
640: "file:", name,
641: FamErrlist[FAMErrno]);
642:
1.1.1.2 ! misho 643: fam_dir_entry_free(&sc->fam, fam_dir);
! 644: fam_dir = NULL;
1.1 misho 645: } else {
646: int osize = 0;
647:
1.1.1.2 ! misho 648: if (sc->dirs) {
1.1 misho 649: osize = sc->dirs->size;
650: }
651:
652: sc->dirs = splaytree_insert(sc->dirs, dir_ndx, fam_dir);
1.1.1.2 ! misho 653: force_assert(sc->dirs);
! 654: force_assert(sc->dirs->data == fam_dir);
! 655: force_assert(osize == (sc->dirs->size - 1));
1.1 misho 656: }
657: } else {
658: fam_dir = dir_node->data;
659: }
660:
661: /* bind the fam_fc to the stat() cache entry */
662:
663: if (fam_dir) {
664: sce->dir_version = fam_dir->version;
665: }
666: }
667: #endif
668:
669: *ret_sce = sce;
670:
671: return HANDLER_GO_ON;
672: }
673:
674: /**
675: * remove stat() from cache which havn't been stat()ed for
676: * more than 10 seconds
677: *
678: *
679: * walk though the stat-cache, collect the ids which are too old
680: * and remove them in a second loop
681: */
682:
683: static int stat_cache_tag_old_entries(server *srv, splay_tree *t, int *keys, size_t *ndx) {
684: stat_cache_entry *sce;
685:
686: if (!t) return 0;
687:
688: stat_cache_tag_old_entries(srv, t->left, keys, ndx);
689: stat_cache_tag_old_entries(srv, t->right, keys, ndx);
690:
691: sce = t->data;
692:
693: if (srv->cur_ts - sce->stat_ts > 2) {
694: keys[(*ndx)++] = t->key;
695: }
696:
697: return 0;
698: }
699:
700: int stat_cache_trigger_cleanup(server *srv) {
701: stat_cache *sc;
702: size_t max_ndx = 0, i;
703: int *keys;
704:
705: sc = srv->stat_cache;
706:
707: if (!sc->files) return 0;
708:
1.1.1.2 ! misho 709: keys = calloc(1, sizeof(int) * sc->files->size);
1.1 misho 710:
711: stat_cache_tag_old_entries(srv, sc->files, keys, &max_ndx);
712:
713: for (i = 0; i < max_ndx; i++) {
714: int ndx = keys[i];
715: splay_tree *node;
716:
717: sc->files = splaytree_splay(sc->files, ndx);
718:
719: node = sc->files;
720:
721: if (node && (node->key == ndx)) {
722: #ifdef DEBUG_STAT_CACHE
723: size_t j;
724: int osize = splaytree_size(sc->files);
725: stat_cache_entry *sce = node->data;
726: #endif
727: stat_cache_entry_free(node->data);
728: sc->files = splaytree_delete(sc->files, ndx);
729:
730: #ifdef DEBUG_STAT_CACHE
731: for (j = 0; j < ctrl.used; j++) {
732: if (ctrl.ptr[j] == ndx) {
733: ctrl.ptr[j] = ctrl.ptr[--ctrl.used];
734: break;
735: }
736: }
737:
1.1.1.2 ! misho 738: force_assert(osize - 1 == splaytree_size(sc->files));
1.1 misho 739: #endif
740: }
741: }
742:
743: free(keys);
744:
745: return 0;
746: }