1: #include "first.h"
2:
3: /**
4: * the network chunk-API
5: *
6: *
7: */
8:
9: #include "chunk.h"
10: #include "base.h"
11: #include "log.h"
12:
13: #include <sys/types.h>
14: #include <sys/stat.h>
15: #include "sys-mmap.h"
16:
17: #include <stdlib.h>
18: #include <fcntl.h>
19: #include <unistd.h>
20:
21: #include <stdio.h>
22: #include <errno.h>
23: #include <string.h>
24:
25: /* default 1MB, upper limit 128MB */
26: #define DEFAULT_TEMPFILE_SIZE (1 * 1024 * 1024)
27: #define MAX_TEMPFILE_SIZE (128 * 1024 * 1024)
28:
29: static array *chunkqueue_default_tempdirs = NULL;
30: static unsigned int chunkqueue_default_tempfile_size = DEFAULT_TEMPFILE_SIZE;
31:
32: chunkqueue *chunkqueue_init(void) {
33: chunkqueue *cq;
34:
35: cq = calloc(1, sizeof(*cq));
36: force_assert(NULL != cq);
37:
38: cq->first = NULL;
39: cq->last = NULL;
40:
41: cq->unused = NULL;
42:
43: cq->tempdirs = chunkqueue_default_tempdirs;
44: cq->upload_temp_file_size = chunkqueue_default_tempfile_size;
45:
46: return cq;
47: }
48:
49: static chunk *chunk_init(void) {
50: chunk *c;
51:
52: c = calloc(1, sizeof(*c));
53: force_assert(NULL != c);
54:
55: c->type = MEM_CHUNK;
56: c->mem = buffer_init();
57: c->file.name = buffer_init();
58: c->file.start = c->file.length = c->file.mmap.offset = 0;
59: c->file.fd = -1;
60: c->file.mmap.start = MAP_FAILED;
61: c->file.mmap.length = 0;
62: c->file.is_temp = 0;
63: c->offset = 0;
64: c->next = NULL;
65:
66: return c;
67: }
68:
69: static void chunk_reset(chunk *c) {
70: if (NULL == c) return;
71:
72: c->type = MEM_CHUNK;
73:
74: buffer_reset(c->mem);
75:
76: if (c->file.is_temp && !buffer_string_is_empty(c->file.name)) {
77: unlink(c->file.name->ptr);
78: }
79:
80: buffer_reset(c->file.name);
81:
82: if (c->file.fd != -1) {
83: close(c->file.fd);
84: c->file.fd = -1;
85: }
86: if (MAP_FAILED != c->file.mmap.start) {
87: munmap(c->file.mmap.start, c->file.mmap.length);
88: c->file.mmap.start = MAP_FAILED;
89: }
90: c->file.start = c->file.length = c->file.mmap.offset = 0;
91: c->file.mmap.length = 0;
92: c->file.is_temp = 0;
93: c->offset = 0;
94: c->next = NULL;
95: }
96:
97: static void chunk_free(chunk *c) {
98: if (NULL == c) return;
99:
100: chunk_reset(c);
101:
102: buffer_free(c->mem);
103: buffer_free(c->file.name);
104:
105: free(c);
106: }
107:
108: static off_t chunk_remaining_length(const chunk *c) {
109: off_t len = 0;
110: switch (c->type) {
111: case MEM_CHUNK:
112: len = buffer_string_length(c->mem);
113: break;
114: case FILE_CHUNK:
115: len = c->file.length;
116: break;
117: default:
118: force_assert(c->type == MEM_CHUNK || c->type == FILE_CHUNK);
119: break;
120: }
121: force_assert(c->offset <= len);
122: return len - c->offset;
123: }
124:
125: void chunkqueue_free(chunkqueue *cq) {
126: chunk *c, *pc;
127:
128: if (NULL == cq) return;
129:
130: for (c = cq->first; c; ) {
131: pc = c;
132: c = c->next;
133: chunk_free(pc);
134: }
135:
136: for (c = cq->unused; c; ) {
137: pc = c;
138: c = c->next;
139: chunk_free(pc);
140: }
141:
142: free(cq);
143: }
144:
145: static void chunkqueue_push_unused_chunk(chunkqueue *cq, chunk *c) {
146: force_assert(NULL != cq && NULL != c);
147:
148: /* keep at max 4 chunks in the 'unused'-cache */
149: if (cq->unused_chunks > 4) {
150: chunk_free(c);
151: } else {
152: chunk_reset(c);
153: c->next = cq->unused;
154: cq->unused = c;
155: cq->unused_chunks++;
156: }
157: }
158:
159: static chunk *chunkqueue_get_unused_chunk(chunkqueue *cq) {
160: chunk *c;
161:
162: force_assert(NULL != cq);
163:
164: /* check if we have a unused chunk */
165: if (0 == cq->unused) {
166: c = chunk_init();
167: } else {
168: /* take the first element from the list (a stack) */
169: c = cq->unused;
170: cq->unused = c->next;
171: c->next = NULL;
172: cq->unused_chunks--;
173: }
174:
175: return c;
176: }
177:
178: static void chunkqueue_prepend_chunk(chunkqueue *cq, chunk *c) {
179: c->next = cq->first;
180: cq->first = c;
181:
182: if (NULL == cq->last) {
183: cq->last = c;
184: }
185: cq->bytes_in += chunk_remaining_length(c);
186: }
187:
188: static void chunkqueue_append_chunk(chunkqueue *cq, chunk *c) {
189: c->next = NULL;
190: if (cq->last) {
191: cq->last->next = c;
192: }
193: cq->last = c;
194:
195: if (NULL == cq->first) {
196: cq->first = c;
197: }
198: cq->bytes_in += chunk_remaining_length(c);
199: }
200:
201: void chunkqueue_reset(chunkqueue *cq) {
202: chunk *cur = cq->first;
203:
204: cq->first = cq->last = NULL;
205:
206: while (NULL != cur) {
207: chunk *next = cur->next;
208: chunkqueue_push_unused_chunk(cq, cur);
209: cur = next;
210: }
211:
212: cq->bytes_in = 0;
213: cq->bytes_out = 0;
214: cq->tempdir_idx = 0;
215: }
216:
217: void chunkqueue_append_file_fd(chunkqueue *cq, buffer *fn, int fd, off_t offset, off_t len) {
218: chunk *c;
219:
220: if (0 == len) {
221: close(fd);
222: return;
223: }
224:
225: c = chunkqueue_get_unused_chunk(cq);
226:
227: c->type = FILE_CHUNK;
228:
229: buffer_copy_buffer(c->file.name, fn);
230: c->file.start = offset;
231: c->file.length = len;
232: c->file.fd = fd;
233: c->offset = 0;
234:
235: chunkqueue_append_chunk(cq, c);
236: }
237:
238: void chunkqueue_append_file(chunkqueue *cq, buffer *fn, off_t offset, off_t len) {
239: chunk *c;
240:
241: if (0 == len) return;
242:
243: c = chunkqueue_get_unused_chunk(cq);
244:
245: c->type = FILE_CHUNK;
246:
247: buffer_copy_buffer(c->file.name, fn);
248: c->file.start = offset;
249: c->file.length = len;
250: c->offset = 0;
251:
252: chunkqueue_append_chunk(cq, c);
253: }
254:
255: void chunkqueue_append_buffer(chunkqueue *cq, buffer *mem) {
256: chunk *c;
257:
258: if (buffer_string_is_empty(mem)) return;
259:
260: c = chunkqueue_get_unused_chunk(cq);
261: c->type = MEM_CHUNK;
262: force_assert(NULL != c->mem);
263: buffer_move(c->mem, mem);
264:
265: chunkqueue_append_chunk(cq, c);
266: }
267:
268: void chunkqueue_prepend_buffer(chunkqueue *cq, buffer *mem) {
269: chunk *c;
270:
271: if (buffer_string_is_empty(mem)) return;
272:
273: c = chunkqueue_get_unused_chunk(cq);
274: c->type = MEM_CHUNK;
275: force_assert(NULL != c->mem);
276: buffer_move(c->mem, mem);
277:
278: chunkqueue_prepend_chunk(cq, c);
279: }
280:
281:
282: void chunkqueue_append_mem(chunkqueue *cq, const char * mem, size_t len) {
283: chunk *c;
284:
285: if (0 == len) return;
286:
287: c = chunkqueue_get_unused_chunk(cq);
288: c->type = MEM_CHUNK;
289: buffer_copy_string_len(c->mem, mem, len);
290:
291: chunkqueue_append_chunk(cq, c);
292: }
293:
294:
295: void chunkqueue_append_chunkqueue(chunkqueue *cq, chunkqueue *src) {
296: if (src == NULL || NULL == src->first) return;
297:
298: if (NULL == cq->first) {
299: cq->first = src->first;
300: } else {
301: cq->last->next = src->first;
302: }
303: cq->last = src->last;
304: cq->bytes_in += (src->bytes_in - src->bytes_out);
305:
306: src->first = NULL;
307: src->last = NULL;
308: src->bytes_out = src->bytes_in;
309: }
310:
311:
312: void chunkqueue_get_memory(chunkqueue *cq, char **mem, size_t *len, size_t min_size, size_t alloc_size) {
313: static const size_t REALLOC_MAX_SIZE = 256;
314: chunk *c;
315: buffer *b;
316: char *dummy_mem;
317: size_t dummy_len;
318:
319: force_assert(NULL != cq);
320: if (NULL == mem) mem = &dummy_mem;
321: if (NULL == len) len = &dummy_len;
322:
323: /* default values: */
324: if (0 == min_size) min_size = 1024;
325: if (0 == alloc_size) alloc_size = 4096;
326: if (alloc_size < min_size) alloc_size = min_size;
327:
328: if (NULL != cq->last && MEM_CHUNK == cq->last->type) {
329: size_t have;
330:
331: b = cq->last->mem;
332: have = buffer_string_space(b);
333:
334: /* unused buffer: allocate space */
335: if (buffer_string_is_empty(b)) {
336: buffer_string_prepare_copy(b, alloc_size);
337: have = buffer_string_space(b);
338: }
339: /* if buffer is really small just make it bigger */
340: else if (have < min_size && b->size <= REALLOC_MAX_SIZE) {
341: size_t cur_len = buffer_string_length(b);
342: size_t new_size = cur_len + min_size, append;
343: if (new_size < alloc_size) new_size = alloc_size;
344:
345: append = new_size - cur_len;
346: if (append >= min_size) {
347: buffer_string_prepare_append(b, append);
348: have = buffer_string_space(b);
349: }
350: }
351:
352: /* return pointer into existing buffer if large enough */
353: if (have >= min_size) {
354: *mem = b->ptr + buffer_string_length(b);
355: *len = have;
356: return;
357: }
358: }
359:
360: /* allocate new chunk */
361: c = chunkqueue_get_unused_chunk(cq);
362: c->type = MEM_CHUNK;
363: chunkqueue_append_chunk(cq, c);
364:
365: b = c->mem;
366: buffer_string_prepare_append(b, alloc_size);
367:
368: *mem = b->ptr + buffer_string_length(b);
369: *len = buffer_string_space(b);
370: }
371:
372: void chunkqueue_use_memory(chunkqueue *cq, size_t len) {
373: buffer *b;
374:
375: force_assert(NULL != cq);
376: force_assert(NULL != cq->last && MEM_CHUNK == cq->last->type);
377: b = cq->last->mem;
378:
379: if (len > 0) {
380: buffer_commit(b, len);
381: cq->bytes_in += len;
382: } else if (buffer_string_is_empty(b)) {
383: /* unused buffer: can't remove chunk easily from
384: * end of list, so just reset the buffer
385: */
386: buffer_reset(b);
387: }
388: }
389:
390: void chunkqueue_set_tempdirs_default (array *tempdirs, unsigned int upload_temp_file_size) {
391: chunkqueue_default_tempdirs = tempdirs;
392: chunkqueue_default_tempfile_size
393: = (0 == upload_temp_file_size) ? DEFAULT_TEMPFILE_SIZE
394: : (upload_temp_file_size > MAX_TEMPFILE_SIZE) ? MAX_TEMPFILE_SIZE
395: : upload_temp_file_size;
396: }
397:
398: #if 0
399: void chunkqueue_set_tempdirs(chunkqueue *cq, array *tempdirs, unsigned int upload_temp_file_size) {
400: force_assert(NULL != cq);
401: cq->tempdirs = tempdirs;
402: cq->upload_temp_file_size
403: = (0 == upload_temp_file_size) ? DEFAULT_TEMPFILE_SIZE
404: : (upload_temp_file_size > MAX_TEMPFILE_SIZE) ? MAX_TEMPFILE_SIZE
405: : upload_temp_file_size;
406: cq->tempdir_idx = 0;
407: }
408: #endif
409:
410: void chunkqueue_steal(chunkqueue *dest, chunkqueue *src, off_t len) {
411: while (len > 0) {
412: chunk *c = src->first;
413: off_t clen = 0, use;
414:
415: if (NULL == c) break;
416:
417: clen = chunk_remaining_length(c);
418: if (0 == clen) {
419: /* drop empty chunk */
420: src->first = c->next;
421: if (c == src->last) src->last = NULL;
422: chunkqueue_push_unused_chunk(src, c);
423: continue;
424: }
425:
426: use = len >= clen ? clen : len;
427: len -= use;
428:
429: if (use == clen) {
430: /* move complete chunk */
431: src->first = c->next;
432: if (c == src->last) src->last = NULL;
433:
434: chunkqueue_append_chunk(dest, c);
435: } else {
436: /* partial chunk with length "use" */
437:
438: switch (c->type) {
439: case MEM_CHUNK:
440: chunkqueue_append_mem(dest, c->mem->ptr + c->offset, use);
441: break;
442: case FILE_CHUNK:
443: /* tempfile flag is in "last" chunk after the split */
444: chunkqueue_append_file(dest, c->file.name, c->file.start + c->offset, use);
445: break;
446: }
447:
448: c->offset += use;
449: force_assert(0 == len);
450: }
451:
452: src->bytes_out += use;
453: }
454: }
455:
456: static chunk *chunkqueue_get_append_tempfile(chunkqueue *cq) {
457: chunk *c;
458: buffer *template = buffer_init_string("/var/tmp/lighttpd-upload-XXXXXX");
459: int fd = -1;
460:
461: if (cq->tempdirs && cq->tempdirs->used) {
462: /* we have several tempdirs, only if all of them fail we jump out */
463:
464: for (errno = EIO; cq->tempdir_idx < cq->tempdirs->used; ++cq->tempdir_idx) {
465: data_string *ds = (data_string *)cq->tempdirs->data[cq->tempdir_idx];
466:
467: buffer_copy_buffer(template, ds->value);
468: buffer_append_slash(template);
469: buffer_append_string_len(template, CONST_STR_LEN("lighttpd-upload-XXXXXX"));
470:
471: if (-1 != (fd = mkstemp(template->ptr))) break;
472: }
473: } else {
474: fd = mkstemp(template->ptr);
475: }
476:
477: if (fd < 0) {
478: buffer_free(template);
479: return NULL;
480: }
481:
482: c = chunkqueue_get_unused_chunk(cq);
483: c->type = FILE_CHUNK;
484: c->file.fd = fd;
485: c->file.is_temp = 1;
486: buffer_copy_buffer(c->file.name, template);
487: c->file.length = 0;
488:
489: chunkqueue_append_chunk(cq, c);
490:
491: buffer_free(template);
492:
493: return c;
494: }
495:
496: static void chunkqueue_remove_empty_chunks(chunkqueue *cq);
497:
498: int chunkqueue_append_mem_to_tempfile(server *srv, chunkqueue *dest, const char *mem, size_t len) {
499: chunk *dst_c;
500: ssize_t written;
501:
502: do {
503: /*
504: * if the last chunk is
505: * - smaller than dest->upload_temp_file_size
506: * - not read yet (offset == 0)
507: * -> append to it (so it might actually become larger than dest->upload_temp_file_size)
508: * otherwise
509: * -> create a new chunk
510: *
511: * */
512:
513: dst_c = dest->last;
514: if (NULL != dst_c
515: && FILE_CHUNK == dst_c->type
516: && dst_c->file.is_temp
517: && dst_c->file.fd >= 0
518: && 0 == dst_c->offset) {
519: /* ok, take the last chunk for our job */
520:
521: if (dst_c->file.length >= (off_t)dest->upload_temp_file_size) {
522: /* the chunk is too large now, close it */
523: int rc = close(dst_c->file.fd);
524: dst_c->file.fd = -1;
525: if (0 != rc) {
526: log_error_write(srv, __FILE__, __LINE__, "sbss",
527: "close() temp-file", dst_c->file.name, "failed:",
528: strerror(errno));
529: return -1;
530: }
531: dst_c = NULL;
532: }
533: } else {
534: dst_c = NULL;
535: }
536:
537: if (NULL == dst_c && NULL == (dst_c = chunkqueue_get_append_tempfile(dest))) {
538: /* we don't have file to write to,
539: * EACCES might be one reason.
540: *
541: * Instead of sending 500 we send 413 and say the request is too large
542: */
543:
544: log_error_write(srv, __FILE__, __LINE__, "ss",
545: "opening temp-file failed:", strerror(errno));
546:
547: return -1;
548: }
549:
550: written = write(dst_c->file.fd, mem, len);
551:
552: if ((size_t) written == len) {
553: dst_c->file.length += len;
554: dest->bytes_in += len;
555:
556: return 0;
557: } else if (written >= 0) {
558: /*(assume EINTR if partial write and retry write();
559: * retry write() might fail with ENOSPC if no more space on volume)*/
560: dest->bytes_in += written;
561: mem += written;
562: len -= (size_t)written;
563: dst_c->file.length += (size_t)written;
564: /* continue; retry */
565: } else if (errno == EINTR) {
566: /* continue; retry */
567: } else {
568: int retry = (errno == ENOSPC && dest->tempdirs && ++dest->tempdir_idx < dest->tempdirs->used);
569: if (!retry) {
570: log_error_write(srv, __FILE__, __LINE__, "sbs",
571: "write() temp-file", dst_c->file.name, "failed:",
572: strerror(errno));
573: }
574:
575: if (0 == chunk_remaining_length(dst_c)) {
576: /*(remove empty chunk and unlink tempfile)*/
577: chunkqueue_remove_empty_chunks(dest);
578: } else {/*(close tempfile; avoid later attempts to append)*/
579: int rc = close(dst_c->file.fd);
580: dst_c->file.fd = -1;
581: if (0 != rc) {
582: log_error_write(srv, __FILE__, __LINE__, "sbss",
583: "close() temp-file", dst_c->file.name, "failed:",
584: strerror(errno));
585: return -1;
586: }
587: }
588: if (!retry) return -1;
589:
590: /* continue; retry */
591: }
592:
593: } while (dst_c);
594:
595: return -1; /*(not reached)*/
596: }
597:
598: int chunkqueue_steal_with_tempfiles(server *srv, chunkqueue *dest, chunkqueue *src, off_t len) {
599: while (len > 0) {
600: chunk *c = src->first;
601: off_t clen = 0, use;
602:
603: if (NULL == c) break;
604:
605: clen = chunk_remaining_length(c);
606: if (0 == clen) {
607: /* drop empty chunk */
608: src->first = c->next;
609: if (c == src->last) src->last = NULL;
610: chunkqueue_push_unused_chunk(src, c);
611: continue;
612: }
613:
614: use = (len >= clen) ? clen : len;
615: len -= use;
616:
617: switch (c->type) {
618: case FILE_CHUNK:
619: if (use == clen) {
620: /* move complete chunk */
621: src->first = c->next;
622: if (c == src->last) src->last = NULL;
623: chunkqueue_append_chunk(dest, c);
624: } else {
625: /* partial chunk with length "use" */
626: /* tempfile flag is in "last" chunk after the split */
627: chunkqueue_append_file(dest, c->file.name, c->file.start + c->offset, use);
628:
629: c->offset += use;
630: force_assert(0 == len);
631: }
632: break;
633:
634: case MEM_CHUNK:
635: /* store "use" bytes from memory chunk in tempfile */
636: if (0 != chunkqueue_append_mem_to_tempfile(srv, dest, c->mem->ptr + c->offset, use)) {
637: return -1;
638: }
639:
640: if (use == clen) {
641: /* finished chunk */
642: src->first = c->next;
643: if (c == src->last) src->last = NULL;
644: chunkqueue_push_unused_chunk(src, c);
645: } else {
646: /* partial chunk */
647: c->offset += use;
648: force_assert(0 == len);
649: }
650: break;
651: }
652:
653: src->bytes_out += use;
654: }
655:
656: return 0;
657: }
658:
659: off_t chunkqueue_length(chunkqueue *cq) {
660: off_t len = 0;
661: chunk *c;
662:
663: for (c = cq->first; c; c = c->next) {
664: len += chunk_remaining_length(c);
665: }
666:
667: return len;
668: }
669:
670: int chunkqueue_is_empty(chunkqueue *cq) {
671: return NULL == cq->first;
672: }
673:
674: void chunkqueue_mark_written(chunkqueue *cq, off_t len) {
675: off_t written = len;
676: chunk *c;
677: force_assert(len >= 0);
678:
679: for (c = cq->first; NULL != c; c = cq->first) {
680: off_t c_len = chunk_remaining_length(c);
681:
682: if (0 == written && 0 != c_len) break; /* no more finished chunks */
683:
684: if (written >= c_len) { /* chunk got finished */
685: c->offset += c_len;
686: written -= c_len;
687:
688: cq->first = c->next;
689: if (c == cq->last) cq->last = NULL;
690:
691: chunkqueue_push_unused_chunk(cq, c);
692: } else { /* partial chunk */
693: c->offset += written;
694: written = 0;
695: break; /* chunk not finished */
696: }
697: }
698:
699: force_assert(0 == written);
700: cq->bytes_out += len;
701: }
702:
703: void chunkqueue_remove_finished_chunks(chunkqueue *cq) {
704: chunk *c;
705:
706: for (c = cq->first; c; c = cq->first) {
707: if (0 != chunk_remaining_length(c)) break; /* not finished yet */
708:
709: cq->first = c->next;
710: if (c == cq->last) cq->last = NULL;
711:
712: chunkqueue_push_unused_chunk(cq, c);
713: }
714: }
715:
716: static void chunkqueue_remove_empty_chunks(chunkqueue *cq) {
717: chunk *c;
718: chunkqueue_remove_finished_chunks(cq);
719: if (chunkqueue_is_empty(cq)) return;
720:
721: for (c = cq->first; c->next; c = c->next) {
722: if (0 == chunk_remaining_length(c->next)) {
723: chunk *empty = c->next;
724: c->next = empty->next;
725: if (empty == cq->last) cq->last = c;
726:
727: chunkqueue_push_unused_chunk(cq, empty);
728: }
729: }
730: }
FreeBSD-CVSweb <freebsd-cvsweb@FreeBSD.org>
![[BACK]](/icons/cvsweb/back.gif){kind=icon}
Return to chunk.c CVS log ![[TXT]](/icons/cvsweb/text.gif){kind=icon}
![[DIR]](/icons/cvsweb/dir.gif){kind=icon}
Up to [ELWIX - Embedded LightWeight unIX -] / embedaddon / lighttpd / src