I've worked on data sets with over 10+ gigs of data. I was provided the data in 1-2 gig text files, both delimited and fixed width.
Since I did not need all data at one time, and given my particular data access needs, I found it optimal to keep the data stored as the original text files and grab out the data as needed using programs that could read the delimiters or by position in the line depending upon the file format as well as perform simple functions such as summing several fields durring the data grab.
Here's a quick example of the type of program I'm speaking of:
clear
CLOSE ALL
set talk off
cutoff={8/31/2002}
delimchar="^"
totalfieldstoget=8
fieldnum1=1 &&acct key
fieldnum2=8 &&cont date
fieldnum3=33 &&cred lim
fieldnum4=35 &&cycl delq
fieldnum5=55 &&state
fieldnum6=48 &&gross balance
fieldnum7=58 &&period enddate
fieldnum8=81 &&comp principal balance
for numberoffiles= 1 to 5
putcmd=""
for x=totalfieldstoget to 1 step -1
putcmd="Valfieldnum"+alltr(str(x))+"+delimchar+"+putcmd
endfor
putcmd="lcString="+left(putcmd,len(putcmd)-11)
varinfile="cards"+alltr(str(numberoffiles))+".txt"
varoutfile="cards"+alltr(str(numberoffiles))+"_out.txt"
lnhandle= fopen(varinfile)
lnNewFile= fcreate(varoutfile)
do while not Feof(lnHandle)
lcString = "^"+fgets(lnHandle,4000)
lcString = strtran(lcString,"','","^"

lcString = chrtran(lcString," '",""

lcString = chrtran(lcString,chr(44),""

IF LEN(ALLTRIM(CHRtran(lcString," ",""

))>0
for x=1 to totalfieldstoget
cmd1=" SUBSTR(lcString,AT(delimchar,lcString,fieldnum"+alltr(str(x))+"

,2)=delimchar+delimchar=.t."
if &cmd1
cmd2="Valfieldnum"+alltr(str(x))+"=''"
&cmd2
endif
cmd3=" SUBSTR(lcString,AT(delimchar,lcString,fieldnum"+alltr(str(x))+"

,2)=delimchar+delimchar=.f."
if &cmd3
cmd4="Valfieldnum"+alltr(str(x))+"=SUBSTR(lcString,AT(delimchar,lcString,fieldnum"+alltr(str(x))+"

+1,(AT(delimchar,lcString,fieldnum"+alltr(str(x))+"+1)-1)-(AT(delimchar,lcString,fieldnum"+alltr(str(x))+"

))"
&cmd4
endif
cmd5="right(alltr(Valfieldnum"+alltr(str(x))+"

,1)='-'"
if &cmd5
cmd6="Valfieldnum"+alltr(str(x))+"='-'+(left(Valfieldnum"+alltr(str(x))+",len(Valfieldnum"+alltr(str(x))+"

-1))"
&cmd6
endif
endfor
&putcmd
= fput(lnNewFile,lcString)
ENDIF
enddo
=fclose(lnHandle)
close all
endfor
?TIME()
create table tempdata (loanid c(30), contdate c(8),creditlim n(9,2),cycdelq n(3),;
state c(2), currbal n(10,2), prdenddate c(8), compprin n(10,2), age n(5))
copy to combo type fox2
use combo
for numberoffiles= 1 to 5
cmd="APPEND FROM cards"+alltr(str(numberoffiles))+"_out.txt DELIMITED WITH CHARACTER ^"
&cmd
endfor
return
repl all age with iif(val(contdate)>0,(val(left(prdenddate,4))*12+val(substr(prdenddate,5,2)))-(val(left(contdate,4))*12+val(substr(contdate,5,2))),0)
calc sum(principal) to a1
calc sum(adminrecv) to b1
calc avg(currbal) to a
calc avg(creditlim) to b
sum currbal to cbal
sum(age*currbal)/cbal to wage
set talk on
\sum principal balance
?a1
\sum finance and admin rec
?b1
\avg gross balance
?a
\avg credit limit
?b
\avg gross balance over avg credit limit
?a/b*100
\wt avg age
?wage