Return to ngx_output_chain.c CVS log | Up to [ELWIX - Embedded LightWeight unIX -] / embedaddon / nginx / src / core |
1.1 ! misho 1: ! 2: /* ! 3: * Copyright (C) Igor Sysoev ! 4: * Copyright (C) Nginx, Inc. ! 5: */ ! 6: ! 7: ! 8: #include <ngx_config.h> ! 9: #include <ngx_core.h> ! 10: #include <ngx_event.h> ! 11: ! 12: ! 13: #if 0 ! 14: #define NGX_SENDFILE_LIMIT 4096 ! 15: #endif ! 16: ! 17: /* ! 18: * When DIRECTIO is enabled FreeBSD, Solaris, and MacOSX read directly ! 19: * to an application memory from a device if parameters are aligned ! 20: * to device sector boundary (512 bytes). They fallback to usual read ! 21: * operation if the parameters are not aligned. ! 22: * Linux allows DIRECTIO only if the parameters are aligned to a filesystem ! 23: * sector boundary, otherwise it returns EINVAL. The sector size is ! 24: * usually 512 bytes, however, on XFS it may be 4096 bytes. ! 25: */ ! 26: ! 27: #define NGX_NONE 1 ! 28: ! 29: ! 30: static ngx_inline ngx_int_t ! 31: ngx_output_chain_as_is(ngx_output_chain_ctx_t *ctx, ngx_buf_t *buf); ! 32: static ngx_int_t ngx_output_chain_add_copy(ngx_pool_t *pool, ! 33: ngx_chain_t **chain, ngx_chain_t *in); ! 34: static ngx_int_t ngx_output_chain_align_file_buf(ngx_output_chain_ctx_t *ctx, ! 35: off_t bsize); ! 36: static ngx_int_t ngx_output_chain_get_buf(ngx_output_chain_ctx_t *ctx, ! 37: off_t bsize); ! 38: static ngx_int_t ngx_output_chain_copy_buf(ngx_output_chain_ctx_t *ctx); ! 39: ! 40: ! 41: ngx_int_t ! 42: ngx_output_chain(ngx_output_chain_ctx_t *ctx, ngx_chain_t *in) ! 43: { ! 44: off_t bsize; ! 45: ngx_int_t rc, last; ! 46: ngx_chain_t *cl, *out, **last_out; ! 47: ! 48: if (ctx->in == NULL && ctx->busy == NULL) { ! 49: ! 50: /* ! 51: * the short path for the case when the ctx->in and ctx->busy chains ! 52: * are empty, the incoming chain is empty too or has the single buf ! 53: * that does not require the copy ! 54: */ ! 55: ! 56: if (in == NULL) { ! 57: return ctx->output_filter(ctx->filter_ctx, in); ! 58: } ! 59: ! 60: if (in->next == NULL ! 61: #if (NGX_SENDFILE_LIMIT) ! 62: && !(in->buf->in_file && in->buf->file_last > NGX_SENDFILE_LIMIT) ! 63: #endif ! 64: && ngx_output_chain_as_is(ctx, in->buf)) ! 65: { ! 66: return ctx->output_filter(ctx->filter_ctx, in); ! 67: } ! 68: } ! 69: ! 70: /* add the incoming buf to the chain ctx->in */ ! 71: ! 72: if (in) { ! 73: if (ngx_output_chain_add_copy(ctx->pool, &ctx->in, in) == NGX_ERROR) { ! 74: return NGX_ERROR; ! 75: } ! 76: } ! 77: ! 78: out = NULL; ! 79: last_out = &out; ! 80: last = NGX_NONE; ! 81: ! 82: for ( ;; ) { ! 83: ! 84: #if (NGX_HAVE_FILE_AIO) ! 85: if (ctx->aio) { ! 86: return NGX_AGAIN; ! 87: } ! 88: #endif ! 89: ! 90: while (ctx->in) { ! 91: ! 92: /* ! 93: * cycle while there are the ctx->in bufs ! 94: * and there are the free output bufs to copy in ! 95: */ ! 96: ! 97: bsize = ngx_buf_size(ctx->in->buf); ! 98: ! 99: if (bsize == 0 && !ngx_buf_special(ctx->in->buf)) { ! 100: ! 101: ngx_log_error(NGX_LOG_ALERT, ctx->pool->log, 0, ! 102: "zero size buf in output " ! 103: "t:%d r:%d f:%d %p %p-%p %p %O-%O", ! 104: ctx->in->buf->temporary, ! 105: ctx->in->buf->recycled, ! 106: ctx->in->buf->in_file, ! 107: ctx->in->buf->start, ! 108: ctx->in->buf->pos, ! 109: ctx->in->buf->last, ! 110: ctx->in->buf->file, ! 111: ctx->in->buf->file_pos, ! 112: ctx->in->buf->file_last); ! 113: ! 114: ngx_debug_point(); ! 115: ! 116: ctx->in = ctx->in->next; ! 117: ! 118: continue; ! 119: } ! 120: ! 121: if (ngx_output_chain_as_is(ctx, ctx->in->buf)) { ! 122: ! 123: /* move the chain link to the output chain */ ! 124: ! 125: cl = ctx->in; ! 126: ctx->in = cl->next; ! 127: ! 128: *last_out = cl; ! 129: last_out = &cl->next; ! 130: cl->next = NULL; ! 131: ! 132: continue; ! 133: } ! 134: ! 135: if (ctx->buf == NULL) { ! 136: ! 137: rc = ngx_output_chain_align_file_buf(ctx, bsize); ! 138: ! 139: if (rc == NGX_ERROR) { ! 140: return NGX_ERROR; ! 141: } ! 142: ! 143: if (rc != NGX_OK) { ! 144: ! 145: if (ctx->free) { ! 146: ! 147: /* get the free buf */ ! 148: ! 149: cl = ctx->free; ! 150: ctx->buf = cl->buf; ! 151: ctx->free = cl->next; ! 152: ! 153: ngx_free_chain(ctx->pool, cl); ! 154: ! 155: } else if (out || ctx->allocated == ctx->bufs.num) { ! 156: ! 157: break; ! 158: ! 159: } else if (ngx_output_chain_get_buf(ctx, bsize) != NGX_OK) { ! 160: return NGX_ERROR; ! 161: } ! 162: } ! 163: } ! 164: ! 165: rc = ngx_output_chain_copy_buf(ctx); ! 166: ! 167: if (rc == NGX_ERROR) { ! 168: return rc; ! 169: } ! 170: ! 171: if (rc == NGX_AGAIN) { ! 172: if (out) { ! 173: break; ! 174: } ! 175: ! 176: return rc; ! 177: } ! 178: ! 179: /* delete the completed buf from the ctx->in chain */ ! 180: ! 181: if (ngx_buf_size(ctx->in->buf) == 0) { ! 182: ctx->in = ctx->in->next; ! 183: } ! 184: ! 185: cl = ngx_alloc_chain_link(ctx->pool); ! 186: if (cl == NULL) { ! 187: return NGX_ERROR; ! 188: } ! 189: ! 190: cl->buf = ctx->buf; ! 191: cl->next = NULL; ! 192: *last_out = cl; ! 193: last_out = &cl->next; ! 194: ctx->buf = NULL; ! 195: } ! 196: ! 197: if (out == NULL && last != NGX_NONE) { ! 198: ! 199: if (ctx->in) { ! 200: return NGX_AGAIN; ! 201: } ! 202: ! 203: return last; ! 204: } ! 205: ! 206: last = ctx->output_filter(ctx->filter_ctx, out); ! 207: ! 208: if (last == NGX_ERROR || last == NGX_DONE) { ! 209: return last; ! 210: } ! 211: ! 212: ngx_chain_update_chains(ctx->pool, &ctx->free, &ctx->busy, &out, ! 213: ctx->tag); ! 214: last_out = &out; ! 215: } ! 216: } ! 217: ! 218: ! 219: static ngx_inline ngx_int_t ! 220: ngx_output_chain_as_is(ngx_output_chain_ctx_t *ctx, ngx_buf_t *buf) ! 221: { ! 222: ngx_uint_t sendfile; ! 223: ! 224: if (ngx_buf_special(buf)) { ! 225: return 1; ! 226: } ! 227: ! 228: if (buf->in_file && buf->file->directio) { ! 229: return 0; ! 230: } ! 231: ! 232: sendfile = ctx->sendfile; ! 233: ! 234: #if (NGX_SENDFILE_LIMIT) ! 235: ! 236: if (buf->in_file && buf->file_pos >= NGX_SENDFILE_LIMIT) { ! 237: sendfile = 0; ! 238: } ! 239: ! 240: #endif ! 241: ! 242: if (!sendfile) { ! 243: ! 244: if (!ngx_buf_in_memory(buf)) { ! 245: return 0; ! 246: } ! 247: ! 248: buf->in_file = 0; ! 249: } ! 250: ! 251: if (ctx->need_in_memory && !ngx_buf_in_memory(buf)) { ! 252: return 0; ! 253: } ! 254: ! 255: if (ctx->need_in_temp && (buf->memory || buf->mmap)) { ! 256: return 0; ! 257: } ! 258: ! 259: return 1; ! 260: } ! 261: ! 262: ! 263: static ngx_int_t ! 264: ngx_output_chain_add_copy(ngx_pool_t *pool, ngx_chain_t **chain, ! 265: ngx_chain_t *in) ! 266: { ! 267: ngx_chain_t *cl, **ll; ! 268: #if (NGX_SENDFILE_LIMIT) ! 269: ngx_buf_t *b, *buf; ! 270: #endif ! 271: ! 272: ll = chain; ! 273: ! 274: for (cl = *chain; cl; cl = cl->next) { ! 275: ll = &cl->next; ! 276: } ! 277: ! 278: while (in) { ! 279: ! 280: cl = ngx_alloc_chain_link(pool); ! 281: if (cl == NULL) { ! 282: return NGX_ERROR; ! 283: } ! 284: ! 285: #if (NGX_SENDFILE_LIMIT) ! 286: ! 287: buf = in->buf; ! 288: ! 289: if (buf->in_file ! 290: && buf->file_pos < NGX_SENDFILE_LIMIT ! 291: && buf->file_last > NGX_SENDFILE_LIMIT) ! 292: { ! 293: /* split a file buf on two bufs by the sendfile limit */ ! 294: ! 295: b = ngx_calloc_buf(pool); ! 296: if (b == NULL) { ! 297: return NGX_ERROR; ! 298: } ! 299: ! 300: ngx_memcpy(b, buf, sizeof(ngx_buf_t)); ! 301: ! 302: if (ngx_buf_in_memory(buf)) { ! 303: buf->pos += (ssize_t) (NGX_SENDFILE_LIMIT - buf->file_pos); ! 304: b->last = buf->pos; ! 305: } ! 306: ! 307: buf->file_pos = NGX_SENDFILE_LIMIT; ! 308: b->file_last = NGX_SENDFILE_LIMIT; ! 309: ! 310: cl->buf = b; ! 311: ! 312: } else { ! 313: cl->buf = buf; ! 314: in = in->next; ! 315: } ! 316: ! 317: #else ! 318: cl->buf = in->buf; ! 319: in = in->next; ! 320: ! 321: #endif ! 322: ! 323: cl->next = NULL; ! 324: *ll = cl; ! 325: ll = &cl->next; ! 326: } ! 327: ! 328: return NGX_OK; ! 329: } ! 330: ! 331: ! 332: static ngx_int_t ! 333: ngx_output_chain_align_file_buf(ngx_output_chain_ctx_t *ctx, off_t bsize) ! 334: { ! 335: size_t size; ! 336: ngx_buf_t *in; ! 337: ! 338: in = ctx->in->buf; ! 339: ! 340: if (in->file == NULL || !in->file->directio) { ! 341: return NGX_DECLINED; ! 342: } ! 343: ! 344: ctx->directio = 1; ! 345: ! 346: size = (size_t) (in->file_pos - (in->file_pos & ~(ctx->alignment - 1))); ! 347: ! 348: if (size == 0) { ! 349: ! 350: if (bsize >= (off_t) ctx->bufs.size) { ! 351: return NGX_DECLINED; ! 352: } ! 353: ! 354: size = (size_t) bsize; ! 355: ! 356: } else { ! 357: size = (size_t) ctx->alignment - size; ! 358: ! 359: if ((off_t) size > bsize) { ! 360: size = (size_t) bsize; ! 361: } ! 362: } ! 363: ! 364: ctx->buf = ngx_create_temp_buf(ctx->pool, size); ! 365: if (ctx->buf == NULL) { ! 366: return NGX_ERROR; ! 367: } ! 368: ! 369: /* ! 370: * we do not set ctx->buf->tag, because we do not want ! 371: * to reuse the buf via ctx->free list ! 372: */ ! 373: ! 374: #if (NGX_HAVE_ALIGNED_DIRECTIO) ! 375: ctx->unaligned = 1; ! 376: #endif ! 377: ! 378: return NGX_OK; ! 379: } ! 380: ! 381: ! 382: static ngx_int_t ! 383: ngx_output_chain_get_buf(ngx_output_chain_ctx_t *ctx, off_t bsize) ! 384: { ! 385: size_t size; ! 386: ngx_buf_t *b, *in; ! 387: ngx_uint_t recycled; ! 388: ! 389: in = ctx->in->buf; ! 390: size = ctx->bufs.size; ! 391: recycled = 1; ! 392: ! 393: if (in->last_in_chain) { ! 394: ! 395: if (bsize < (off_t) size) { ! 396: ! 397: /* ! 398: * allocate a small temp buf for a small last buf ! 399: * or its small last part ! 400: */ ! 401: ! 402: size = (size_t) bsize; ! 403: recycled = 0; ! 404: ! 405: } else if (!ctx->directio ! 406: && ctx->bufs.num == 1 ! 407: && (bsize < (off_t) (size + size / 4))) ! 408: { ! 409: /* ! 410: * allocate a temp buf that equals to a last buf, ! 411: * if there is no directio, the last buf size is lesser ! 412: * than 1.25 of bufs.size and the temp buf is single ! 413: */ ! 414: ! 415: size = (size_t) bsize; ! 416: recycled = 0; ! 417: } ! 418: } ! 419: ! 420: b = ngx_calloc_buf(ctx->pool); ! 421: if (b == NULL) { ! 422: return NGX_ERROR; ! 423: } ! 424: ! 425: if (ctx->directio) { ! 426: ! 427: /* ! 428: * allocate block aligned to a disk sector size to enable ! 429: * userland buffer direct usage conjunctly with directio ! 430: */ ! 431: ! 432: b->start = ngx_pmemalign(ctx->pool, size, (size_t) ctx->alignment); ! 433: if (b->start == NULL) { ! 434: return NGX_ERROR; ! 435: } ! 436: ! 437: } else { ! 438: b->start = ngx_palloc(ctx->pool, size); ! 439: if (b->start == NULL) { ! 440: return NGX_ERROR; ! 441: } ! 442: } ! 443: ! 444: b->pos = b->start; ! 445: b->last = b->start; ! 446: b->end = b->last + size; ! 447: b->temporary = 1; ! 448: b->tag = ctx->tag; ! 449: b->recycled = recycled; ! 450: ! 451: ctx->buf = b; ! 452: ctx->allocated++; ! 453: ! 454: return NGX_OK; ! 455: } ! 456: ! 457: ! 458: static ngx_int_t ! 459: ngx_output_chain_copy_buf(ngx_output_chain_ctx_t *ctx) ! 460: { ! 461: off_t size; ! 462: ssize_t n; ! 463: ngx_buf_t *src, *dst; ! 464: ngx_uint_t sendfile; ! 465: ! 466: src = ctx->in->buf; ! 467: dst = ctx->buf; ! 468: ! 469: size = ngx_buf_size(src); ! 470: size = ngx_min(size, dst->end - dst->pos); ! 471: ! 472: sendfile = ctx->sendfile & !ctx->directio; ! 473: ! 474: #if (NGX_SENDFILE_LIMIT) ! 475: ! 476: if (src->in_file && src->file_pos >= NGX_SENDFILE_LIMIT) { ! 477: sendfile = 0; ! 478: } ! 479: ! 480: #endif ! 481: ! 482: if (ngx_buf_in_memory(src)) { ! 483: ngx_memcpy(dst->pos, src->pos, (size_t) size); ! 484: src->pos += (size_t) size; ! 485: dst->last += (size_t) size; ! 486: ! 487: if (src->in_file) { ! 488: ! 489: if (sendfile) { ! 490: dst->in_file = 1; ! 491: dst->file = src->file; ! 492: dst->file_pos = src->file_pos; ! 493: dst->file_last = src->file_pos + size; ! 494: ! 495: } else { ! 496: dst->in_file = 0; ! 497: } ! 498: ! 499: src->file_pos += size; ! 500: ! 501: } else { ! 502: dst->in_file = 0; ! 503: } ! 504: ! 505: if (src->pos == src->last) { ! 506: dst->flush = src->flush; ! 507: dst->last_buf = src->last_buf; ! 508: dst->last_in_chain = src->last_in_chain; ! 509: } ! 510: ! 511: } else { ! 512: ! 513: #if (NGX_HAVE_ALIGNED_DIRECTIO) ! 514: ! 515: if (ctx->unaligned) { ! 516: if (ngx_directio_off(src->file->fd) == NGX_FILE_ERROR) { ! 517: ngx_log_error(NGX_LOG_ALERT, ctx->pool->log, ngx_errno, ! 518: ngx_directio_off_n " \"%s\" failed", ! 519: src->file->name.data); ! 520: } ! 521: } ! 522: ! 523: #endif ! 524: ! 525: #if (NGX_HAVE_FILE_AIO) ! 526: ! 527: if (ctx->aio_handler) { ! 528: n = ngx_file_aio_read(src->file, dst->pos, (size_t) size, ! 529: src->file_pos, ctx->pool); ! 530: if (n == NGX_AGAIN) { ! 531: ctx->aio_handler(ctx, src->file); ! 532: return NGX_AGAIN; ! 533: } ! 534: ! 535: } else { ! 536: n = ngx_read_file(src->file, dst->pos, (size_t) size, ! 537: src->file_pos); ! 538: } ! 539: #else ! 540: ! 541: n = ngx_read_file(src->file, dst->pos, (size_t) size, src->file_pos); ! 542: ! 543: #endif ! 544: ! 545: #if (NGX_HAVE_ALIGNED_DIRECTIO) ! 546: ! 547: if (ctx->unaligned) { ! 548: ngx_err_t err; ! 549: ! 550: err = ngx_errno; ! 551: ! 552: if (ngx_directio_on(src->file->fd) == NGX_FILE_ERROR) { ! 553: ngx_log_error(NGX_LOG_ALERT, ctx->pool->log, ngx_errno, ! 554: ngx_directio_on_n " \"%s\" failed", ! 555: src->file->name.data); ! 556: } ! 557: ! 558: ngx_set_errno(err); ! 559: ! 560: ctx->unaligned = 0; ! 561: } ! 562: ! 563: #endif ! 564: ! 565: if (n == NGX_ERROR) { ! 566: return (ngx_int_t) n; ! 567: } ! 568: ! 569: if (n != size) { ! 570: ngx_log_error(NGX_LOG_ALERT, ctx->pool->log, 0, ! 571: ngx_read_file_n " read only %z of %O from \"%s\"", ! 572: n, size, src->file->name.data); ! 573: return NGX_ERROR; ! 574: } ! 575: ! 576: dst->last += n; ! 577: ! 578: if (sendfile) { ! 579: dst->in_file = 1; ! 580: dst->file = src->file; ! 581: dst->file_pos = src->file_pos; ! 582: dst->file_last = src->file_pos + n; ! 583: ! 584: } else { ! 585: dst->in_file = 0; ! 586: } ! 587: ! 588: src->file_pos += n; ! 589: ! 590: if (src->file_pos == src->file_last) { ! 591: dst->flush = src->flush; ! 592: dst->last_buf = src->last_buf; ! 593: dst->last_in_chain = src->last_in_chain; ! 594: } ! 595: } ! 596: ! 597: return NGX_OK; ! 598: } ! 599: ! 600: ! 601: ngx_int_t ! 602: ngx_chain_writer(void *data, ngx_chain_t *in) ! 603: { ! 604: ngx_chain_writer_ctx_t *ctx = data; ! 605: ! 606: off_t size; ! 607: ngx_chain_t *cl; ! 608: ngx_connection_t *c; ! 609: ! 610: c = ctx->connection; ! 611: ! 612: for (size = 0; in; in = in->next) { ! 613: ! 614: #if 1 ! 615: if (ngx_buf_size(in->buf) == 0 && !ngx_buf_special(in->buf)) { ! 616: ngx_debug_point(); ! 617: } ! 618: #endif ! 619: ! 620: size += ngx_buf_size(in->buf); ! 621: ! 622: ngx_log_debug2(NGX_LOG_DEBUG_CORE, c->log, 0, ! 623: "chain writer buf fl:%d s:%uO", ! 624: in->buf->flush, ngx_buf_size(in->buf)); ! 625: ! 626: cl = ngx_alloc_chain_link(ctx->pool); ! 627: if (cl == NULL) { ! 628: return NGX_ERROR; ! 629: } ! 630: ! 631: cl->buf = in->buf; ! 632: cl->next = NULL; ! 633: *ctx->last = cl; ! 634: ctx->last = &cl->next; ! 635: } ! 636: ! 637: ngx_log_debug1(NGX_LOG_DEBUG_CORE, c->log, 0, ! 638: "chain writer in: %p", ctx->out); ! 639: ! 640: for (cl = ctx->out; cl; cl = cl->next) { ! 641: ! 642: #if 1 ! 643: if (ngx_buf_size(cl->buf) == 0 && !ngx_buf_special(cl->buf)) { ! 644: ngx_debug_point(); ! 645: } ! 646: ! 647: #endif ! 648: ! 649: size += ngx_buf_size(cl->buf); ! 650: } ! 651: ! 652: if (size == 0 && !c->buffered) { ! 653: return NGX_OK; ! 654: } ! 655: ! 656: ctx->out = c->send_chain(c, ctx->out, ctx->limit); ! 657: ! 658: ngx_log_debug1(NGX_LOG_DEBUG_CORE, c->log, 0, ! 659: "chain writer out: %p", ctx->out); ! 660: ! 661: if (ctx->out == NGX_CHAIN_ERROR) { ! 662: return NGX_ERROR; ! 663: } ! 664: ! 665: if (ctx->out == NULL) { ! 666: ctx->last = &ctx->out; ! 667: ! 668: if (!c->buffered) { ! 669: return NGX_OK; ! 670: } ! 671: } ! 672: ! 673: return NGX_AGAIN; ! 674: }