*! 15dec2010 
program collect
version 11.1 
mata: DoIt()
if ~mi("`mem'") {
	di "{txt:Adjusting memory...}"
	clear
	clear mata
	qui set memory `mem'k
	local mem
	mata: DoIt()
	if ~mi("`mem'") errel Adjusted memory was insufficient...
	}

end 

version 11.1
mata:

void DoIt() {
	class dataset scalar ds
	class dataview scalar dv
	class prefs scalar pr
	
	syntaxl(st_local("0"),(&(paths="anything"),&(ifin="[ifin]")), (&(keep="[k:eep()]"),&(test="[t:est]"),&(nomore="[nom:ore]"),&(gen="[g:enerate()]"),&(app="[app:end]")))
	parts=columnize(paths,";")
	files=J(0,1,"")
	for (p=1;p<=length(parts);p++) files=files\multipath(parts[p],".dta")
	files=uniqrows(files)
	if (length(files)==0) errel(sprintf("No files found: %s\n",concat(parts,";")))
	
	matched=ds.combinevars(keep,files,app)
	
	if (strlen(gen)) {
		ds.nvars=ds.nvars+1
		ds.vnames=ds.vnames,gen
		ds.varlabels=ds.varlabels,"collect file id"
		ds.sirtypes=ds.sirtypes,"i"
		ds.bytes=ds.bytes,1
		ds.formats=ds.formats,"%3.0f"
		ds.labrefs=ds.labrefs,"collectidlab" //check for dups?!
		fbits=columnize(files,dirsep()) //this bit to use only relevant end of filepaths
		if (rows(fibts)>1) {
			f=1
			while (rows(uniqrows(fbits[,f]))==1) ++f
			if (f<cols(fbits)) fbits=concat(fbits[,f..cols(fbits)],dirsep())
			else fbits=fbits[,f]
			}
		else fbits=fbits[,cols(fbits)]
		fbits="data in memory"\fbits
		ds.labnames=ds.labnames,"collectidlab"
		ds.labtabs=ds.labtabs,&(strofreal(0::rows(files)),fbits)
		}

	printf("{txt:Estimated memory required: %11.0fc k}\n",ds.nobs*sum(ds.bytes)/1000)
	if (ds.nobs*sum(ds.bytes)>c("memory")&!nomore) {
		st_local("mem",strofreal(trunc(ds.nobs*sum(ds.bytes)/1000*1.6),"%12.0f")) //this could be much better: width vs length; absolute size... //however, Stata say to allocate at least 1.5 times, to avoid spurious oom errors...
		return
		}
	
	if (app) {
		ds.overlay()
		beg=st_nobs()+1
		if (strlen(gen)) stata(sprintf("qui replace %s=0",gen))
		st_addobs(ds.nobs-st_nobs())
		}
	else {
		ds.writemem()
		beg=1
		}
	
	for (f=1;f<=rows(files);f++) {
		ds.read("data",files[f])
		if ((xtra=beg+ds.nobs-1-st_nobs())>0) st_addobs(xtra) //ds was estimate 
		map=*matched[f]
		dv.view(beg::beg+ds.nobs-1,map[1,])
		dv.copyin(ds,map[2,])
		if (strlen(gen)) st_store(beg::beg+ds.nobs-1,gen,J(ds.nobs,1,f))
		beg=beg+ds.nobs
		}
	if (beg<=st_nobs()) st_dropobsin((beg,st_nobs()))
	
	pr.make("recentcollected","path")
	pr.body=files //consider adding other info
	pr.write()
	
	}

end