1: /*****************************************************************************
2: *
3: * This example source code introduces a c library buffered I/O interface to
4: * URL reads it supports fopen(), fread(), fgets(), feof(), fclose(),
5: * rewind(). Supported functions have identical prototypes to their normal c
6: * lib namesakes and are preceaded by url_ .
7: *
8: * Using this code you can replace your program's fopen() with url_fopen()
9: * and fread() with url_fread() and it become possible to read remote streams
10: * instead of (only) local files. Local files (ie those that can be directly
11: * fopened) will drop back to using the underlying clib implementations
12: *
13: * See the main() function at the bottom that shows an app that retrieves from
14: * a specified url using fgets() and fread() and saves as two output files.
15: *
16: * Copyright (c) 2003 - 2019 Simtec Electronics
17: *
18: * Re-implemented by Vincent Sanders <vince@kyllikki.org> with extensive
19: * reference to original curl example code
20: *
21: * Redistribution and use in source and binary forms, with or without
22: * modification, are permitted provided that the following conditions
23: * are met:
24: * 1. Redistributions of source code must retain the above copyright
25: * notice, this list of conditions and the following disclaimer.
26: * 2. Redistributions in binary form must reproduce the above copyright
27: * notice, this list of conditions and the following disclaimer in the
28: * documentation and/or other materials provided with the distribution.
29: * 3. The name of the author may not be used to endorse or promote products
30: * derived from this software without specific prior written permission.
31: *
32: * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
33: * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
34: * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
35: * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
36: * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
37: * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
38: * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
39: * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
40: * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
41: * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
42: *
43: * This example requires libcurl 7.9.7 or later.
44: */
45: /* <DESC>
46: * implements an fopen() abstraction allowing reading from URLs
47: * </DESC>
48: */
49:
50: #include <stdio.h>
51: #include <string.h>
52: #ifndef WIN32
53: # include <sys/time.h>
54: #endif
55: #include <stdlib.h>
56: #include <errno.h>
57:
58: #include <curl/curl.h>
59:
60: enum fcurl_type_e {
61: CFTYPE_NONE = 0,
62: CFTYPE_FILE = 1,
63: CFTYPE_CURL = 2
64: };
65:
66: struct fcurl_data
67: {
68: enum fcurl_type_e type; /* type of handle */
69: union {
70: CURL *curl;
71: FILE *file;
72: } handle; /* handle */
73:
74: char *buffer; /* buffer to store cached data*/
75: size_t buffer_len; /* currently allocated buffers length */
76: size_t buffer_pos; /* end of data in buffer*/
77: int still_running; /* Is background url fetch still in progress */
78: };
79:
80: typedef struct fcurl_data URL_FILE;
81:
82: /* exported functions */
83: URL_FILE *url_fopen(const char *url, const char *operation);
84: int url_fclose(URL_FILE *file);
85: int url_feof(URL_FILE *file);
86: size_t url_fread(void *ptr, size_t size, size_t nmemb, URL_FILE *file);
87: char *url_fgets(char *ptr, size_t size, URL_FILE *file);
88: void url_rewind(URL_FILE *file);
89:
90: /* we use a global one for convenience */
91: static CURLM *multi_handle;
92:
93: /* curl calls this routine to get more data */
94: static size_t write_callback(char *buffer,
95: size_t size,
96: size_t nitems,
97: void *userp)
98: {
99: char *newbuff;
100: size_t rembuff;
101:
102: URL_FILE *url = (URL_FILE *)userp;
103: size *= nitems;
104:
105: rembuff = url->buffer_len - url->buffer_pos; /* remaining space in buffer */
106:
107: if(size > rembuff) {
108: /* not enough space in buffer */
109: newbuff = realloc(url->buffer, url->buffer_len + (size - rembuff));
110: if(newbuff == NULL) {
111: fprintf(stderr, "callback buffer grow failed\n");
112: size = rembuff;
113: }
114: else {
115: /* realloc succeeded increase buffer size*/
116: url->buffer_len += size - rembuff;
117: url->buffer = newbuff;
118: }
119: }
120:
121: memcpy(&url->buffer[url->buffer_pos], buffer, size);
122: url->buffer_pos += size;
123:
124: return size;
125: }
126:
127: /* use to attempt to fill the read buffer up to requested number of bytes */
128: static int fill_buffer(URL_FILE *file, size_t want)
129: {
130: fd_set fdread;
131: fd_set fdwrite;
132: fd_set fdexcep;
133: struct timeval timeout;
134: int rc;
135: CURLMcode mc; /* curl_multi_fdset() return code */
136:
137: /* only attempt to fill buffer if transactions still running and buffer
138: * doesn't exceed required size already
139: */
140: if((!file->still_running) || (file->buffer_pos > want))
141: return 0;
142:
143: /* attempt to fill buffer */
144: do {
145: int maxfd = -1;
146: long curl_timeo = -1;
147:
148: FD_ZERO(&fdread);
149: FD_ZERO(&fdwrite);
150: FD_ZERO(&fdexcep);
151:
152: /* set a suitable timeout to fail on */
153: timeout.tv_sec = 60; /* 1 minute */
154: timeout.tv_usec = 0;
155:
156: curl_multi_timeout(multi_handle, &curl_timeo);
157: if(curl_timeo >= 0) {
158: timeout.tv_sec = curl_timeo / 1000;
159: if(timeout.tv_sec > 1)
160: timeout.tv_sec = 1;
161: else
162: timeout.tv_usec = (curl_timeo % 1000) * 1000;
163: }
164:
165: /* get file descriptors from the transfers */
166: mc = curl_multi_fdset(multi_handle, &fdread, &fdwrite, &fdexcep, &maxfd);
167:
168: if(mc != CURLM_OK) {
169: fprintf(stderr, "curl_multi_fdset() failed, code %d.\n", mc);
170: break;
171: }
172:
173: /* On success the value of maxfd is guaranteed to be >= -1. We call
174: select(maxfd + 1, ...); specially in case of (maxfd == -1) there are
175: no fds ready yet so we call select(0, ...) --or Sleep() on Windows--
176: to sleep 100ms, which is the minimum suggested value in the
177: curl_multi_fdset() doc. */
178:
179: if(maxfd == -1) {
180: #ifdef _WIN32
181: Sleep(100);
182: rc = 0;
183: #else
184: /* Portable sleep for platforms other than Windows. */
185: struct timeval wait = { 0, 100 * 1000 }; /* 100ms */
186: rc = select(0, NULL, NULL, NULL, &wait);
187: #endif
188: }
189: else {
190: /* Note that on some platforms 'timeout' may be modified by select().
191: If you need access to the original value save a copy beforehand. */
192: rc = select(maxfd + 1, &fdread, &fdwrite, &fdexcep, &timeout);
193: }
194:
195: switch(rc) {
196: case -1:
197: /* select error */
198: break;
199:
200: case 0:
201: default:
202: /* timeout or readable/writable sockets */
203: curl_multi_perform(multi_handle, &file->still_running);
204: break;
205: }
206: } while(file->still_running && (file->buffer_pos < want));
207: return 1;
208: }
209:
210: /* use to remove want bytes from the front of a files buffer */
211: static int use_buffer(URL_FILE *file, size_t want)
212: {
213: /* sort out buffer */
214: if(file->buffer_pos <= want) {
215: /* ditch buffer - write will recreate */
216: free(file->buffer);
217: file->buffer = NULL;
218: file->buffer_pos = 0;
219: file->buffer_len = 0;
220: }
221: else {
222: /* move rest down make it available for later */
223: memmove(file->buffer,
224: &file->buffer[want],
225: (file->buffer_pos - want));
226:
227: file->buffer_pos -= want;
228: }
229: return 0;
230: }
231:
232: URL_FILE *url_fopen(const char *url, const char *operation)
233: {
234: /* this code could check for URLs or types in the 'url' and
235: basically use the real fopen() for standard files */
236:
237: URL_FILE *file;
238: (void)operation;
239:
240: file = calloc(1, sizeof(URL_FILE));
241: if(!file)
242: return NULL;
243:
244: file->handle.file = fopen(url, operation);
245: if(file->handle.file)
246: file->type = CFTYPE_FILE; /* marked as URL */
247:
248: else {
249: file->type = CFTYPE_CURL; /* marked as URL */
250: file->handle.curl = curl_easy_init();
251:
252: curl_easy_setopt(file->handle.curl, CURLOPT_URL, url);
253: curl_easy_setopt(file->handle.curl, CURLOPT_WRITEDATA, file);
254: curl_easy_setopt(file->handle.curl, CURLOPT_VERBOSE, 0L);
255: curl_easy_setopt(file->handle.curl, CURLOPT_WRITEFUNCTION, write_callback);
256:
257: if(!multi_handle)
258: multi_handle = curl_multi_init();
259:
260: curl_multi_add_handle(multi_handle, file->handle.curl);
261:
262: /* lets start the fetch */
263: curl_multi_perform(multi_handle, &file->still_running);
264:
265: if((file->buffer_pos == 0) && (!file->still_running)) {
266: /* if still_running is 0 now, we should return NULL */
267:
268: /* make sure the easy handle is not in the multi handle anymore */
269: curl_multi_remove_handle(multi_handle, file->handle.curl);
270:
271: /* cleanup */
272: curl_easy_cleanup(file->handle.curl);
273:
274: free(file);
275:
276: file = NULL;
277: }
278: }
279: return file;
280: }
281:
282: int url_fclose(URL_FILE *file)
283: {
284: int ret = 0;/* default is good return */
285:
286: switch(file->type) {
287: case CFTYPE_FILE:
288: ret = fclose(file->handle.file); /* passthrough */
289: break;
290:
291: case CFTYPE_CURL:
292: /* make sure the easy handle is not in the multi handle anymore */
293: curl_multi_remove_handle(multi_handle, file->handle.curl);
294:
295: /* cleanup */
296: curl_easy_cleanup(file->handle.curl);
297: break;
298:
299: default: /* unknown or supported type - oh dear */
300: ret = EOF;
301: errno = EBADF;
302: break;
303: }
304:
305: free(file->buffer);/* free any allocated buffer space */
306: free(file);
307:
308: return ret;
309: }
310:
311: int url_feof(URL_FILE *file)
312: {
313: int ret = 0;
314:
315: switch(file->type) {
316: case CFTYPE_FILE:
317: ret = feof(file->handle.file);
318: break;
319:
320: case CFTYPE_CURL:
321: if((file->buffer_pos == 0) && (!file->still_running))
322: ret = 1;
323: break;
324:
325: default: /* unknown or supported type - oh dear */
326: ret = -1;
327: errno = EBADF;
328: break;
329: }
330: return ret;
331: }
332:
333: size_t url_fread(void *ptr, size_t size, size_t nmemb, URL_FILE *file)
334: {
335: size_t want;
336:
337: switch(file->type) {
338: case CFTYPE_FILE:
339: want = fread(ptr, size, nmemb, file->handle.file);
340: break;
341:
342: case CFTYPE_CURL:
343: want = nmemb * size;
344:
345: fill_buffer(file, want);
346:
347: /* check if there's data in the buffer - if not fill_buffer()
348: * either errored or EOF */
349: if(!file->buffer_pos)
350: return 0;
351:
352: /* ensure only available data is considered */
353: if(file->buffer_pos < want)
354: want = file->buffer_pos;
355:
356: /* xfer data to caller */
357: memcpy(ptr, file->buffer, want);
358:
359: use_buffer(file, want);
360:
361: want = want / size; /* number of items */
362: break;
363:
364: default: /* unknown or supported type - oh dear */
365: want = 0;
366: errno = EBADF;
367: break;
368:
369: }
370: return want;
371: }
372:
373: char *url_fgets(char *ptr, size_t size, URL_FILE *file)
374: {
375: size_t want = size - 1;/* always need to leave room for zero termination */
376: size_t loop;
377:
378: switch(file->type) {
379: case CFTYPE_FILE:
380: ptr = fgets(ptr, (int)size, file->handle.file);
381: break;
382:
383: case CFTYPE_CURL:
384: fill_buffer(file, want);
385:
386: /* check if there's data in the buffer - if not fill either errored or
387: * EOF */
388: if(!file->buffer_pos)
389: return NULL;
390:
391: /* ensure only available data is considered */
392: if(file->buffer_pos < want)
393: want = file->buffer_pos;
394:
395: /*buffer contains data */
396: /* look for newline or eof */
397: for(loop = 0; loop < want; loop++) {
398: if(file->buffer[loop] == '\n') {
399: want = loop + 1;/* include newline */
400: break;
401: }
402: }
403:
404: /* xfer data to caller */
405: memcpy(ptr, file->buffer, want);
406: ptr[want] = 0;/* always null terminate */
407:
408: use_buffer(file, want);
409:
410: break;
411:
412: default: /* unknown or supported type - oh dear */
413: ptr = NULL;
414: errno = EBADF;
415: break;
416: }
417:
418: return ptr;/*success */
419: }
420:
421: void url_rewind(URL_FILE *file)
422: {
423: switch(file->type) {
424: case CFTYPE_FILE:
425: rewind(file->handle.file); /* passthrough */
426: break;
427:
428: case CFTYPE_CURL:
429: /* halt transaction */
430: curl_multi_remove_handle(multi_handle, file->handle.curl);
431:
432: /* restart */
433: curl_multi_add_handle(multi_handle, file->handle.curl);
434:
435: /* ditch buffer - write will recreate - resets stream pos*/
436: free(file->buffer);
437: file->buffer = NULL;
438: file->buffer_pos = 0;
439: file->buffer_len = 0;
440:
441: break;
442:
443: default: /* unknown or supported type - oh dear */
444: break;
445: }
446: }
447:
448: #define FGETSFILE "fgets.test"
449: #define FREADFILE "fread.test"
450: #define REWINDFILE "rewind.test"
451:
452: /* Small main program to retrieve from a url using fgets and fread saving the
453: * output to two test files (note the fgets method will corrupt binary files if
454: * they contain 0 chars */
455: int main(int argc, char *argv[])
456: {
457: URL_FILE *handle;
458: FILE *outf;
459:
460: size_t nread;
461: char buffer[256];
462: const char *url;
463:
464: if(argc < 2)
465: url = "http://192.168.7.3/testfile";/* default to testurl */
466: else
467: url = argv[1];/* use passed url */
468:
469: /* copy from url line by line with fgets */
470: outf = fopen(FGETSFILE, "wb+");
471: if(!outf) {
472: perror("couldn't open fgets output file\n");
473: return 1;
474: }
475:
476: handle = url_fopen(url, "r");
477: if(!handle) {
478: printf("couldn't url_fopen() %s\n", url);
479: fclose(outf);
480: return 2;
481: }
482:
483: while(!url_feof(handle)) {
484: url_fgets(buffer, sizeof(buffer), handle);
485: fwrite(buffer, 1, strlen(buffer), outf);
486: }
487:
488: url_fclose(handle);
489:
490: fclose(outf);
491:
492:
493: /* Copy from url with fread */
494: outf = fopen(FREADFILE, "wb+");
495: if(!outf) {
496: perror("couldn't open fread output file\n");
497: return 1;
498: }
499:
500: handle = url_fopen("testfile", "r");
501: if(!handle) {
502: printf("couldn't url_fopen() testfile\n");
503: fclose(outf);
504: return 2;
505: }
506:
507: do {
508: nread = url_fread(buffer, 1, sizeof(buffer), handle);
509: fwrite(buffer, 1, nread, outf);
510: } while(nread);
511:
512: url_fclose(handle);
513:
514: fclose(outf);
515:
516:
517: /* Test rewind */
518: outf = fopen(REWINDFILE, "wb+");
519: if(!outf) {
520: perror("couldn't open fread output file\n");
521: return 1;
522: }
523:
524: handle = url_fopen("testfile", "r");
525: if(!handle) {
526: printf("couldn't url_fopen() testfile\n");
527: fclose(outf);
528: return 2;
529: }
530:
531: nread = url_fread(buffer, 1, sizeof(buffer), handle);
532: fwrite(buffer, 1, nread, outf);
533: url_rewind(handle);
534:
535: buffer[0]='\n';
536: fwrite(buffer, 1, 1, outf);
537:
538: nread = url_fread(buffer, 1, sizeof(buffer), handle);
539: fwrite(buffer, 1, nread, outf);
540:
541: url_fclose(handle);
542:
543: fclose(outf);
544:
545: return 0;/* all done */
546: }
FreeBSD-CVSweb <freebsd-cvsweb@FreeBSD.org>
![[BACK]](/icons/cvsweb/back.gif){kind=icon}
Return to fopen.c CVS log ![[TXT]](/icons/cvsweb/text.gif){kind=icon}
![[DIR]](/icons/cvsweb/dir.gif){kind=icon}
Up to [ELWIX - Embedded LightWeight unIX -] / embedaddon / curl / docs / examples