read line by line in the most efficient way *platform specific*
Doing read(2) is not very good because of the fragmentation of the lines (e.g. if you read 1000 chars, the last line may start at offset 990 and need 50 chars beyond the buffer).
Better to use fgets [recommended] or mmap [YMMV]. Here are examples of each. Caveat: compiles, but not tested, and doesn't do much error checking
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <fcntl.h>
#include <unistd.h>
#include <sys/stat.h>
#include <sys/mman.h>
typedef struct {
unsigned int lidx;
char *lncontent;
} FileL;
// read in file lines using fgets
FileL *
getfileline_fgets(char *file)
{
FILE *xf;
char *cp;
int len;
int linecnt;
int linemax;
FileL *linelist;
FileL *line;
char lbuf[50000];
xf = fopen(file,"r");
linecnt = 0;
linemax = 0;
linelist = NULL;
while (1) {
cp = fgets(lbuf,sizeof(lbuf),xf);
if (cp == NULL)
break;
len = strlen(lbuf);
// strip newline from string [or not]
#if 1
if (len > 0) {
cp = &lbuf[len - 1];
if (*cp == '\n') {
*cp = 0;
--len;
}
}
#endif
if ((linecnt + 1) > linemax) {
linemax += 100;
linelist = realloc(linelist,linemax * sizeof(FileL));
}
line = &linelist[linecnt];
line->lidx = linecnt++;
cp = malloc(len + 1);
memcpy(cp,lbuf,len + 1);
line->lncontent = cp;
}
fclose(xf);
// trim to maximum used
linelist = realloc(linelist,linecnt * sizeof(FileL));
return linelist;
}
// read in file lines by mmap to entire file
FileL *
getfilelines_mmap(char *file)
{
int fd;
char *lhs;
char *rhs;
char *cp;
int len;
int linecnt;
int linemax;
FileL *linelist;
FileL *line;
struct stat st;
char *fbuf;
char cbuf[50000];
fd = open(file,O_RDONLY);
fstat(fd,&st);
fbuf = mmap(NULL,st.st_size,PROT_READ,MAP_PRIVATE,fd,0);
linecnt = 0;
linemax = 0;
linelist = NULL;
lhs = fbuf;
rhs = fbuf;
for (lhs = fbuf; lhs < &fbuf[st.st_size]; lhs = rhs + 1) {
rhs = strchr(lhs,'\n');
// NOTE: does _not_ handle case of malformed text file that has _no_
// newline on last line
if (rhs == NULL)
break;
len = rhs - lhs;
// strip newline from string [or not]
#if 1
if (len > 0)
--len;
#endif
if ((linecnt + 1) > linemax) {
linemax += 100;
linelist = realloc(linelist,linemax * sizeof(FileL));
}
line = &linelist[linecnt];
line->lidx = linecnt++;
cp = malloc(len + 1);
memcpy(cp,lhs,len);
cp[len] = 0;
line->lncontent = cp;
}
munmap(fbuf,st.st_size);
close(fd);
// trim to maximum used
linelist = realloc(linelist,linecnt * sizeof(FileL));
return linelist;
}
UPDATE
You wanted benchmarks. Well, you're gonna get 'em. Generated 167GB of random text data, spanning 140 files. The L: is # of lines, W: is max width, and shows filesize in MB. Times are in nanoseconds. Note that factors vary, but looks like mmap wins.
23:39:35.528333425 NEWDAY 11/09/15
23:39:35.528333425 ph: starting 23107 ...
23:39:35.528868198 ph: ARGV fastreadgo ...
F001: L:324255 W:2097 324.086MB
368297556 fgets
189180143 mmap
F002: L:329608 W:2822 443.649MB
475989122 fgets
248517335 mmap
F003: L:401476 W:6186 1185.270MB
1206999411 fgets
657703847 mmap
F004: L:729379 W:9350 3253.185MB
3199692871 fgets
1776602082 mmap
F005: L:85857 W:5185 212.599MB
223489564 fgets
122404608 mmap
F006: L:62871 W:5418 162.384MB
167640768 fgets
93127042 mmap
F007: L:298836 W:1083 154.481MB
196584474 fgets
100582134 mmap
F008: L:221513 W:2732 288.694MB
322105867 fgets
164965547 mmap
F009: L:420815 W:8906 1789.672MB
1801309998 fgets
961136893 mmap
F010: L:126712 W:8251 498.905MB
499274233 fgets
275901635 mmap
F011: L:443166 W:8822 1865.753MB
1839816883 fgets
1001882651 mmap
F012: L:385632 W:2162 398.467MB
467223648 fgets
248126909 mmap
F013: L:629448 W:4413 1324.616MB
1432284339 fgets
777593198 mmap
F014: L:510357 W:7313 1779.348MB
1919309671 fgets
1079111734 mmap
F015: L:188434 W:1254 112.922MB
152367682 fgets
78959769 mmap
F016: L:82139 W:4355 170.586MB
193117015 fgets
105417805 mmap
F017: L:389499 W:9063 1681.805MB
1730894028 fgets
913789253 mmap
F018: L:992849 W:3265 1547.875MB
1685006767 fgets
875256226 mmap
F019: L:931502 W:9647 4285.883MB
11181005402 fgets
2361255543 mmap
F020: L:266047 W:7454 946.298MB
955772708 fgets
537059554 mmap
F021: L:572709 W:67 18.835MB
86539501 fgets
43437303 mmap
F022: L:68373 W:3042 98.684MB
110325296 fgets
57538963 mmap
F023: L:651839 W:2006 624.153MB
706094723 fgets
369122560 mmap
F024: L:414658 W:6482 1284.202MB
1294352248 fgets
700279769 mmap
F025: L:984554 W:3441 1616.269MB
1742233370 fgets
903755131 mmap
F026: L:527629 W:3214 808.812MB
872660092 fgets
465403685 mmap
F027: L:572103 W:6219 1696.582MB
1758562312 fgets
933024466 mmap
F028: L:793354 W:5967 2255.653MB
2341754885 fgets
1251633414 mmap
F029: L:690669 W:389 128.888MB
230036016 fgets
119381427 mmap
F030: L:902519 W:8182 3523.415MB
6665490426 fgets
1930049511 mmap
F031: L:179482 W:2361 201.850MB
225333697 fgets
120424715 mmap
F032: L:342396 W:4135 675.885MB
706219203 fgets
379974402 mmap
F033: L:762237 W:4000 1455.780MB
1535236381 fgets
805977762 mmap
F034: L:421947 W:8289 1669.038MB
1686877811 fgets
900813641 mmap
F035: L:367349 W:5829 1022.373MB
1051584165 fgets
566680706 mmap
F036: L:433973 W:5064 1049.724MB
1097920811 fgets
584855289 mmap
F037: L:615918 W:9152 2686.372MB
2743719787 fgets
1468536802 mmap
F038: L:365187 W:1564 272.829MB
326368364 fgets
171071840 mmap
F039: L:61305 W:477 14.002MB
22945438 fgets
11949833 mmap
F040: L:396788 W:8576 1622.049MB
1633217001 fgets
884460205 mmap
F041: L:245326 W:5068 592.450MB
610530077 fgets
328366102 mmap
F042: L:986409 W:9174 4313.608MB
17048484450 fgets
2413375121 mmap
F043: L:367968 W:9703 1703.785MB
1677764299 fgets
922735827 mmap
F044: L:630679 W:9763 2942.911MB
4742195305 fgets
1585438052 mmap
F045: L:397072 W:7717 1459.554MB
1533634531 fgets
860518182 mmap
F046: L:918129 W:9127 3996.179MB
10259712214 fgets
2171550789 mmap
F047: L:770706 W:2720 999.584MB
1097599308 fgets
604894013 mmap
F048: L:472462 W:5011 1127.896MB
1164186449 fgets
621979909 mmap
F049: L:301834 W:4456 642.703MB
664420452 fgets
354255131 mmap
F050: L:213878 W:2913 297.159MB
321396955 fgets
168664579 mmap
F051: L:549950 W:1681 441.842MB
510553173 fgets
260455948 mmap
F052: L:63502 W:8785 267.074MB
265697002 fgets
142457939 mmap
F053: L:880396 W:6821 2864.595MB
3769485430 fgets
1591318886 mmap
F054: L:180543 W:9055 779.566MB
773462618 fgets
428627500 mmap
F055: L:964409 W:8454 3884.437MB
9085108760 fgets
2149540695 mmap
F056: L:675120 W:8912 2872.781MB
2885159527 fgets
1559580604 mmap
F057: L:345151 W:4157 684.052MB
724456228 fgets
387170980 mmap
F058: L:69114 W:4585 150.535MB
157447952 fgets
84782951 mmap
F059: L:304627 W:9441 1370.777MB
1376517664 fgets
739170571 mmap
F060: L:799770 W:3145 1200.762MB
1304001986 fgets
679163462 mmap
F061: L:808699 W:6544 2523.949MB
2590924710 fgets
1385627164 mmap
F062: L:270082 W:313 40.592MB
78777863 fgets
40733146 mmap
F063: L:308883 W:333 49.262MB
93696361 fgets
48580067 mmap
F064: L:237002 W:2618 296.446MB
347315129 fgets
178078149 mmap
F065: L:279040 W:1341 178.685MB
217230537 fgets
113291912 mmap
F066: L:809386 W:2808 1085.734MB
1177480987 fgets
615248653 mmap
F067: L:279448 W:8560 1140.280MB
1151044788 fgets
614662533 mmap
F068: L:80012 W:7441 283.334MB
286915203 fgets
158077955 mmap
F069: L:366808 W:7197 1260.521MB
1292679736 fgets
696686301 mmap
F070: L:272693 W:9275 1206.527MB
1220763889 fgets
658383413 mmap
F071: L:792609 W:1419 537.088MB
645760162 fgets
334886975 mmap
F072: L:742523 W:8640 3059.604MB
5711688133 fgets
1665879727 mmap
F073: L:583753 W:2992 833.759MB
910037328 fgets
483847376 mmap
F074: L:252560 W:7178 864.593MB
868625985 fgets
471770777 mmap
F075: L:154327 W:7026 515.619MB
516135586 fgets
277690063 mmap
F076: L:121839 W:7131 414.684MB
424518600 fgets
230518357 mmap
F077: L:760327 W:1421 515.475MB
622630358 fgets
314592959 mmap
F078: L:907033 W:3485 1508.042MB
1622356297 fgets
845695719 mmap
F079: L:884787 W:7491 3162.774MB
4932864122 fgets
1749065509 mmap
F080: L:432556 W:6039 1245.779MB
1281231973 fgets
693532807 mmap
F081: L:639804 W:6419 1957.747MB
2107303517 fgets
1130299002 mmap
F082: L:388669 W:283 52.804MB
111686630 fgets
57177517 mmap
F083: L:300542 W:1943 278.825MB
336538347 fgets
177494803 mmap
F084: L:941 W:7,3878 1.770MB
2047540 fgets
1347230 mmap
F085: L:85747 W:1841 75.417MB
92274672 fgets
49362653 mmap
F086: L:935559 W:5950 2656.411MB
2734326131 fgets
1487147776 mmap
F087: L:936993 W:1197 535.727MB
672872562 fgets
348765250 mmap
F088: L:409671 W:5235 1023.358MB
1099320520 fgets
606047909 mmap
F089: L:362220 W:5434 938.805MB
991448256 fgets
529093412 mmap
F090: L:628156 W:3682 1103.909MB
1185317812 fgets
637902310 mmap
F091: L:655456 W:6051 1892.574MB
1978859918 fgets
1066368241 mmap
F092: L:356309 W:5946 1012.893MB
1046818030 fgets
562463577 mmap
F093: L:878726 W:2946 1236.162MB
1368885560 fgets
701514499 mmap
F094: L:583863 W:747 208.701MB
293177923 fgets
148045230 mmap
F095: L:51374 W:3752 91.670MB
98830853 fgets
52715699 mmap
F096: L:757271 W:4698 1698.664MB
1790811621 fgets
946452098 mmap
F097: L:665420 W:1814 575.369MB
664290848 fgets
347346293 mmap
F098: L:152806 W:4480 326.336MB
338683910 fgets
185037896 mmap
F099: L:39027 W:2368 44.104MB
49144948 fgets
26701307 mmap
F100: L:896926 W:8209 3513.328MB
7460727008 fgets
1900543480 mmap
F101: L:796628 W:5663 2149.888MB
2207899454 fgets
1187397751 mmap
F102: L:876500 W:1986 831.161MB
934850175 fgets
486626065 mmap
F103: L:188682 W:773 69.722MB
97285527 fgets
48765985 mmap
F104: L:648920 W:9590 2969.446MB
5021968784 fgets
1622268239 mmap
F105: L:827850 W:2123 837.892MB
946063144 fgets
498163978 mmap
F106: L:879828 W:2867 1205.021MB
1304295176 fgets
682155187 mmap
F107: L:970674 W:3830 1771.667MB
1883664162 fgets
989569477 mmap
F108: L:4461 W:5634 11.840MB
12680659 fgets
7159011 mmap
F109: L:477207 W:1067 243.224MB
315370392 fgets
162708299 mmap
F110: L:140308 W:5817 389.132MB
397510757 fgets
216204659 mmap
F111: L:253358 W:4425 534.937MB
559943651 fgets
297109524 mmap
F112: L:903292 W:7989 3441.851MB
7327033977 fgets
1906200470 mmap
F113: L:555989 W:620 164.835MB
245638038 fgets
126559933 mmap
F114: L:596425 W:2330 664.143MB
739017073 fgets
391237002 mmap
F115: L:298147 W:9741 1387.530MB
1363229979 fgets
744420477 mmap
F116: L:180269 W:4522 389.175MB
402702977 fgets
213875684 mmap
F117: L:238597 W:9021 1029.314MB
1033070395 fgets
550442036 mmap
F118: L:183723 W:8705 764.555MB
765959712 fgets
413667801 mmap
F119: L:174802 W:549 45.896MB
70635625 fgets
35721310 mmap
F120: L:883013 W:4666 1963.677MB
2062197751 fgets
1092583730 mmap
F121: L:858995 W:9218 3776.896MB
9278222413 fgets
2309240152 mmap
F122: L:368895 W:5862 1030.174MB
1076473726 fgets
582127460 mmap
F123: L:208043 W:5672 563.889MB
579427255 fgets
310321934 mmap
F124: L:768482 W:4953 1816.657MB
1888233155 fgets
997797932 mmap
F125: L:905425 W:2812 1214.882MB
1394928053 fgets
724059403 mmap
F126: L:54137 W:4690 121.066MB
125124760 fgets
67811537 mmap
F127: L:448100 W:9643 2061.624MB
2066282543 fgets
1126488038 mmap
F128: L:748979 W:2111 754.038MB
854095589 fgets
447406977 mmap
F129: L:611388 W:6954 2026.306MB
2074219917 fgets
1118353849 mmap
F130: L:782834 W:9946 3715.067MB
7338500374 fgets
2029571615 mmap
F131: L:52630 W:7858 197.495MB
200711062 fgets
110759659 mmap
F132: L:930983 W:7363 3270.546MB
3376813502 fgets
1776365395 mmap
F133: L:73216 W:2127 74.344MB
85854537 fgets
46756335 mmap
F134: L:583306 W:2495 694.192MB
766430638 fgets
408095226 mmap
F135: L:877424 W:2964 1241.342MB
1339005805 fgets
702659289 mmap
F136: L:414854 W:5104 1010.006MB
1057372341 fgets
556583887 mmap
F137: L:333176 W:4912 781.109MB
820007572 fgets
435433956 mmap
F138: L:564006 W:6933 1863.574MB
1905024687 fgets
1030574213 mmap
F139: L:829571 W:9152 3622.399MB
7338698902 fgets
2002428493 mmap
F140: L:560210 W:7443 1990.047MB
2012670010 fgets
1098720143 mmap
00:00:58.770988225 NEWDAY 11/10/15
00:00:58.770988225 ph: complete (ELAPSED: 00:21:23.190149545)
Here's the perl script I used to generate the files:
#!/usr/bin/perl
# grpcntgen -- generate test data for fastread algorithms
#
# arguments:
# "-W" - maximum line width
# "-L" - maximum number of lines
# "-T" - number of test files to generate
# "-O" - output file (e.g. foo%.txt)
#
# NOTE: with no arguments or missing arguments will prompt
#pragma pgmlns
# tstgen -- test generation help routines
# gengetstr -- get a string/number
sub gengetstr
{
my($numflg,$opt,$prompt,$lim) = @_;
my($arg);
my($askflg);
my($val);
select(STDOUT);
$| = 1;
{
# search command line for -whatever
foreach $arg (@argv) {
if ($arg =~ /^$opt(.*)$/) {
$val = $1;
if ($numflg && ($val eq "")) {
$val = $lim;
$val //= 1;
}
last;
}
}
last if (defined($val));
$askflg = 1;
while (1) {
printf("Enter ")
if ($numflg != 1);
printf("%s",$prompt);
if ($numflg == 1) {
printf(" (0/1)? ");
}
else {
printf(": ");
}
$val = <STDIN>;
chomp($val);
if ($numflg == 0) {
last if ($val ne "");
next;
}
# an empty response for a number with a maximum means use it
if (($numflg == 2) && ($val eq "") && defined($lim)) {
$val = $lim;
last;
}
next unless ($val =~ /^\d+$/);
$val += 0;
last if ($numflg == 1);
next if ($val <= 0);
last unless (defined($lim));
last if ($val <= $lim);
}
}
unless ($askflg) {
printf("%s: %s\n",$prompt,$val);
}
$val;
}
# genrun -- generate all tests
sub genrun
{
local(@argv) = @_;
local($ofile,$tstmax,$tstproc);
local($tstcur);
local($splitflg);
local($genvbq);
my($sym);
my($numfmt);
my($xfile);
$genvbq = genvbq(\@argv);
$ofile = shift(@argv);
$tstmax = shift(@argv);
$tstproc = shift(@argv);
# split each test into separate file
if ($ofile =~ /%/) {
$splitflg = 1;
$numfmt = sprintf("%d",$tstmax);
$numfmt = length($numfmt);
$numfmt = sprintf("_%%%d.%dd",$numfmt,$numfmt);
$ofile =~ s/%/$numfmt/;
###die("genrun: DEBUG_CAE numfmt='$numfmt' ofile='$ofile'\n");
}
{
last if ($splitflg);
genopen($ofile);
}
for ($tstcur = 1; $tstcur <= $tstmax; ++$tstcur) {
{
last unless ($splitflg);
$xfile = sprintf($ofile,$tstcur);
genopen($xfile);
}
&$tstproc();
{
last unless ($splitflg);
genclose();
}
}
{
last if ($splitflg);
genclose();
}
}
# genvbq -- get options
sub genvbq
{
my($argv) = @_;
my($sym);
my($env);
$env = {};
while (1) {
$sym = $argv->[0];
last unless ($sym =~ s/^-//);
shift(@$argv);
if ($sym =~ /^([^=]+)=(.+)$/) {
($sym,$val) = ($1,$2);
}
else {
$val = 1;
}
$env->{$sym} = $val;
}
$env;
}
# genopen -- open output
sub genopen
{
my($ofile) = @_;
$gen_ofile = $ofile;
{
last if ($genvbq->{"n"});
last if (open($xfdst,">$ofile"));
die("genopen: unable to open '$ofile' -- $!\n");
}
}
# genclose -- close output
sub genclose
{
close($xfdst);
}
# geninit -- initialize for single test
sub geninit
{
undef($genout_lhs);
undef($genout_pre);
}
# genout -- output data
sub genout
{
my($rhs) = @_;
{
if (defined($rhs)) {
last if ((length($genout_pre) + length($genout_lhs) +
length($rhs)) < 78);
}
last if ($genout_lhs eq "");
print($xfdst $genout_pre,$genout_lhs,"\n");
undef($genout_lhs);
}
$genout_lhs .= $rhs
if (defined($rhs));
}
# genrand -- get random number
sub genrand
{
my($lim) = @_;
my($val);
$val = int(rand($lim));
$val += 1;
$val;
}
# genfmtof -- get number format
sub genfmtof
{
my($num) = @_;
my($fmt);
$fmt = sprintf("%d",$num);
$fmt = length($fmt);
$fmt = sprintf(" %%%dd",$fmt);
$fmt;
}
1;
master(@ARGV);
exit(0);
# master -- master control
sub master
{
local(@argv) = @_;
$Wmax = gengetstr(2,"-W","maximum line width");
$Lmax = gengetstr(2,"-L","maximum number of lines / file");
$tstmax = gengetstr(2,"-T","number of tests");
while (1) {
$ofile = gengetstr(0,"-O","output file name");
last if ($ofile =~ /%/);
printf("fastreadgen: filename must have %% in it (e.g. foo%%.txt)\n");
}
genrun($ofile,$tstmax,\&dotest);
}
# dotest -- generate a test
sub dotest
{
my($lno,$chroff);
my($lhs);
$Wlim = genrand($Wmax);
$Llim = genrand($Lmax);
printf("dotest: T=%d/%d W=%d L=%d\n",$tstcur,$tstmax,$Wlim,$Llim);
for ($lno = 1; $lno <= $Llim; ++$lno) {
$Wcur = genrand($Wlim);
$Wcur -= 1;
undef($lhs);
for (; $Wcur > 0; --$Wcur) {
$chroff = genrand(0x7E - 0x20);
$chroff += 0x20;
$chroff -= 1;
$chroff = chr($chroff);
last;
$lhs .= $chroff;
}
$lhs = $chroff x $Wcur;
print($xfdst $lhs,"\n");
}
}