;
;
; -input large fila NAS_arun_listing_nz
; - you can run strmatch on it, but stregex() takes forever.. so
; - using cut, get rid of everything to the left of col 47
;  cut -b 47- NAS_arun_listing_nz >arun_fnames
;  you can run stregex on this file
; -but .. the original file has errors, etc that don't align
;         with col47 so, figure out which lines in NAS_	need to be excluded
; - also, arun switch the ls listing  toward the end so cut  only works for the
;   first part. For the 2nd part just grab NAS_xx
;   split  where ok ne 0 and first  char not '-'
;
; arrays:
; inpall[n]  --aruns large file all info
; inp[m]     --fnames only:
;               m<n
;               cut large file before col 47
;               append end of inpall that only has fnames
;               remove entries with rsynerro,dir names, etc
; par[3,m]   get aruns Type,proj nm from first 2 xx/yy/ of inp
;
; save,fitsnm,fitstype,ftypnm[],file='fits.sav'
;   fitstype[]nm
; fitsNm[p]  = all filenames that end in .fits
; fitstype[p]=    . categorize kind of fits file
;             1-7
; 
; 
goto,dofits
; 
inpfile='NAS_arun_listing_nz'
n=readasciifile(inpfile,inpall)
;
;a=stregex(inpall[0:1e5],".* ([^ ]+)$",/sub,/extr)
;val=strsplit(inpall,/extract)
;
;
; try cleaning it up
;
ok=intarr(n) +1
; --> get rid of  rsync: error
;
valAr=strmatch(inpall,"r*",/fold)
iirsync=where(valar eq 1,cnt)
inpall_rsync=inpall[iirsync]
ok[iirsync]=0
;
; don't remove links.. about 200000  probably needed  for processing
; just flag them with ok=2
valAr=strmatch(inpall,"l*",/fold)
iilink=where(valar eq 1,cnt)
inpall_link=inpall[iilink]
ok[iilink]++ ; links have 2
;
; separate out directory names then exclude
valAr=strmatch(inpall,"d*",/fold)
iidir1=where(valar eq 1,cnt)
valAr=strmatch(inpall,"*/")
iidir2=where((valar eq 1) and (ok ne 0),cnt)
iidir=[iidir1,iidir2]
inpall_dir=inpall[iidir]
ok[iidir]=0
;
; find out where arun switched his ls listing
;
valAr=strmatch(inpall,"-*")
iinewls=where((valar eq 0) and (ok ne 0) and (ok ne 2),cnt)
;
; input aruns file:
;
inpfile='arun_fnames'
n=readasciifile(inpfile,inp)
i0=iinewls[0]  ; where it switches
inpnm=[inp[0L:i0-1],inpall[i0:*]]
ii=where(ok ne 0,cnt)
inpnm=inpnm[ii]
;
;a=stregex(inpnm,"^([^/]+)/([^/]+)",/sub,/ext)
a=stregex(inpnm,"^([^/]+)/",/sub,/ext)
;
; type/nm
;
type=reform(a[1,*])
utype=type[uniq(type,sort(type))]
ntype=n_elements(utype)
for i=0,ntype-1 do print,i," " + utype[i] 
;
; fix the 3 Planetary that have leading space
;
ii=where(a[1,*] eq utype[1],cnt)
print,inpnm[ii]
inpnm[ii]=strtrim(inpnm[ii],1)
;
; redo type/nm
; parse array
pAr=stregex(inpnm,"^([^/]+)/",/sub,/ext)
;
; type/nm
;
type=reform(pAr[1,*])
utype=type[uniq(type,sort(type))]
ntype=n_elements(utype)
for i=0,ntype-1 do print,i," " + utype[i] 
;
; for each type parse generate subdir containing filelist
; also generate html table
;
cntAr=lonarr(ntype)
szArMB=fltarr(ntype)
far=strarr(ntype)
odir="typelist/"
file_mkdir,odir
for itype=1,ntype-1 do begin &$
	ii=where(type eq utype[itype],cnt) &$
	cntAr[itype]=cnt &$
	print,itype,cnt &$
	obase=odir +utype[itype] &$
	ofile=obase+".list" &$
	far[itype]=ofile &$
	openw,lunout,ofile,/get_lun &$
	printf,lunout,inpnm[ii] &$
	free_lun,lunout &$
	fs=file_info(ofile) &$
	szArMB[itype]=fs.size/(2l^20 * 1.) &$
endfor
;
; generate html table
;
htmlfile='typefile.html'
openw,lunout,htmlfile,/get_lun
	lnks='<a moz-do-not-send="true" href="arunslist/'
	lnkm='">'
	lnke='</a>'
	trs='<tr>'
	tre='</tr>'
	tds='<td valign="top">'
	tde='</td>'
	tdes=tde+tds
	for i=1,ntype-1 do begin &$
		ln1=lnks+far[i]+lnkm + utype[i]+lnke &$
		printf,lunout,$
		format='(5x,a,a,a,a,i7,a,f7.3,a,a)',$
			trs,tds,ln1,tdes,cntAr[i],tdes,szArMb[i],tde ,tre  &$
	endfor
    free_lun,lunout
stop
;
; generate uniq proj list, all id's
;  36000.
if 0 then begin
projAr=reform(par[2,*])
uprojar=projar[uniq(projar,sort(projar))]
openw,lunout,'junk.out',/get_lun
printf,lunout,uprojar,format='(a)'
free_lun,lunout
stop
endif
;-------------------------------------------------------------------------
; get all fits files allow compressed files
;
dofits:
val=strmatch(inpnm,"*.fits*")
;val=strmatch(inpnm,"*.fits")
ii=where(val eq 1,cntfits)
fitsnm=inpnm[ii]
parFits=par[*,ii] 
;
a={ind:0,$
   name:''}
ftAr=replicate(a,7) 
i=0
ftAr[i].ind=1
ftAr[i++].name='gpu'
;
ftAr[i].ind=2
ftAr[i++].name='wapp'
;
ftAr[i].ind=3
ftAr[i++].name='mock'
;
ftAr[i].ind=4
ftAr[i++].name='puppi'
;
ftAr[i].ind=5
ftAr[i++].name='galfa'
;
ftAr[i].ind=6
ftAr[i++].name='planetary'
;
ftAr[i].ind=7
ftAr[i++].name='galfacube'

ok=intarr(cntfits) 
;
; gpu
;
j=where(ftAr.name eq 'gpu')
val=strmatch(fitsnm,"*gpu[0-9]*",/fold)
jj=where(val eq 1,cnt)
ok[jj]=ftar[j].ind
;
; wapp
;
j=where(ftAr.name eq 'wapp')
val=strmatch(fitsnm,"*/wapp.*",/fold)
jj=where((val eq 1) and (ok eq 0),cnt)
ok[jj]=ftar[j].ind
;
; mock
;
j=where(ftAr.name eq 'mock')
val=strmatch(fitsnm,"*b?s?g?*")
jj=where((val eq 1) and (ok eq 0),cnt)
ok[jj]=ftar[j].ind
;
; puppi (no gpu)
;
j=where(ftAr.name eq 'puppi')
val=strmatch(fitsnm,"*puppi_*",/fold)
jj=where((val eq 1) and (ok eq 0),cnt)
ok[jj]=ftar[j].ind
;
; galfa 
;
j=where(ftAr.name eq 'galfa')
val=strmatch(fitsnm,"*galfa.*",/fold)
jj=where((val eq 1) and (ok eq 0),cnt)
ok[jj]=ftar[j].ind
;
; galfacube 
;
j=where(ftAr.name eq 'galfacube')
val=strmatch(fitsnm,"*galfacube*",/fold)
jj=where((val eq 1) and (ok eq 0),cnt)
ok[jj]=ftar[j].ind
;
;
; planetary
;
j=where(ftAr.name eq 'planetary')
val=strmatch(fitsnm,"*Planetar*",/fold)
jj=where((val eq 1) and (ok eq 0),cnt)
ok[jj]=ftar[j].ind
;
fitstype=ok
Readme=["fitsNm[n]:strAR filename all fits files",$
        "fitsType[n];intAr integer # what kind of fits file",$
	    "ftAr[m]structAr ind,name for each fits type"]



save,fitsnm,fitstype,ftAr,Readme,file='fits.sav'
;
;
end