;
; inpt fits.sav .. 
; generate subdir for each file type
; the proj that have these files with filelist
; RESTORE: Restored variable: FITSNM.
; RESTORE: Restored variable: FITSTYPE.
; RESTORE: Restored variable: FTAR.
; RESTORE: Restored variable: README.
; --> these look like spectra line, not pulsar
;
odirFits='fitsfiles/'
file_mkdir,odirfits
restore,'fits.sav',/verb
ntypes=n_elements(ftar)
;--------------------------------------------
; wapp files
wapp: 
;
; wapp
ii=where(ftar.name eq 'wapp')
iiwapp=ftar[ii].ind
ii=where(fitstype eq iiwapp,cnt)
;
;
a=stregex(fitsnm[ii],".*/(wapp\..*\.fits.*)",/sub,/ext)
; just the basename
wappnm=reform(a[1,*])
; check
jj=where(wappnm eq '',cntbad)
print,"cntbad:",cntbad
; how many don't  end in .fits
;aa=stregex(wappnm,".*.fits$",/sub,/ext)
;
; split into date, projid
;
a=stregex(wappnm,"wapp\.[^.]+\.([^.]+)",/sub,/ext)
projid=reform(a[1,*])
uprojid=projid[uniq(projid,sort(projid))]
nproj=n_elements(uprojid)
odir=odirfits + "wapp/"
file_mkdir,odir

for i=0,nproj-1 do begin &$
	ii=where(projid eq uprojid[i],cnt) &$
	ofile=odir+ uprojid[i]  &$
	openw,lunout,ofile,/get_lun &$
	printf,lunout,":Nfiles:",cnt &$
	printf,lunout,wappnm[ii],format='(a)' &$
    free_lun,lunout &$
endfor
stop
;--------------------------------------------
; galfa
galfa:
;
; GALFAHI/galfaraw/galfa.20130817.a2124.0000.fits
;
ii=where(ftar.name eq 'galfa')
iigalfa=ftar[ii].ind
ii=where(fitstype eq iigalfa,cnt)
;
; see if all has galfaraw
; .. yes
;val=strmatch(fitsnm[ii],"*galfaraw*")
;jj=where(val eq 0,cnt)
;print,val
;
;
a=stregex(fitsnm[ii],".*/(galfa\..*\.fits.*)",/sub,/ext)
; just the basename
galfanm=reform(a[1,*])
; check
ii=where(galfanm eq '',cnt)
print,"cntbad:",cnt
;
; split into date, projid
;                         date     proj 
a=stregex(galfanm,"galfa\.[^.]+\.([^.]+)",/sub,/ext)
projid=reform(a[1,*])
uprojid=projid[uniq(projid,sort(projid))]
nproj=n_elements(uprojid)
odir=odirfits + "galfa/"
file_mkdir,odir

for i=0,nproj-1 do begin &$
    ii=where(projid eq uprojid[i],cnt) &$
    ofile=odir+ uprojid[i]  &$
    openw,lunout,ofile,/get_lun &$
    printf,lunout,":Nfiles:",cnt &$
    printf,lunout,galfanm[ii],format='(a)' &$
    free_lun,lunout &$
endfor
stop
;
;--------------------------------------------
; galfacube
;ao/AODATA/GALFACUBE1/tiles/G156+26pj_NGC3274.fits.bz2
;ao/AODATA/GALFACUBE1/tiles/GALFA_HI_RA+DEC_108.00+02.35_N.fits.bz2
;ao/AODATA/GALFACUBE1/tiles/GALFA_HI_RA+DEC_108.00+02.35/GALFA_HI_RA+DEC_108.00+02.35_001.fits.bz2
galfacube:
;
;
ii=where(ftar.name eq 'galfacube')
iigalfa=ftar[ii].ind
ii=where(fitstype eq iigalfa,cnt)
;
;
;
a=stregex(fitsnm[ii],".*/([^/]+\.fits.*)",/sub,/ext)
jj=where(a[1,*] ne '',cnt)
; just the basename
galfanm=reform(a[1,*])
; check
ii=where(galfanm eq '',cntbad)
print,"cntbad:",cntbadprocessing:
;
; no projs
odir=odirfits + "galfacube/"
file_mkdir,odir
openw,lunout,odir+"galfacube",/get_lun
printf,lunout,":Nfiles:",cnt &$
printf,lunout,galfanm,format='(a)' 
free_lun,lunout
stop
;
;--------------------------------------------
puppi:
;
; puppi name without gpu.. (have these been combined?)
; filename does not have the proj.. in the path
; 1 / : P3218/puppi_58099_B2016+28_1447_cal_0001.fits
; 2 / :AODATA/P3218/puppi_58107_B2016+28_1654_0002.fits 
; 4 / : arun/p1693/deneva-nrl21/0/4bit_puppi_58025_Strip230_0059.0001.fits
; need to leave complete path since some dups
; 
ii=where(ftar.name eq 'puppi')
iipuppi=ftar[ii].ind
ii=where(fitstype eq iipuppi,cnt)
pupnm=fitsnm[ii]
; make projid same order as pupnm
projid=strarr(cnt)
basenm=strarr(cnt)
;
; 1 / proj at start
;                 proj / basenm
a=stregex(pupnm,"^([^/]+)/([^/]+\.fits.*)",/sub,/ext)
ii=where(a[2,*] ne '',cnt1)
if cnt1 gt 0 then begin &$
 	 projid[ii]=reform(a[1,ii]) &$
     basenm[ii]=reform(a[2,ii]) &$
endif
; AO327/n/basename..
a=stregex(pupnm,"^([^/]+)/[0-9]/([^/]+\.fits.*)",/sub,/ext)
ii=where(a[2,*] ne '',cnt)
projid[ii]=reform(a[1,ii])
basenm[ii]=reform(a[2,ii])
cnt1+=cnt
;
; proj 2nd, basenm last
;                        proj    basenm
a=stregex(pupnm,"^[^/]+/([^/]+).*/([^/]+\.fits.*)",/sub,/ext)
ii=where((a[2,*] ne '') and (projid eq ''),cnt2)
if cnt2 gt 0 then begin &$
projid[ii]=reform(a[1,ii])  &$
basenm[ii]=reform(a[2,ii])  &$
endif
;
; check for duplicate basenm's
;
unm=basenm[uniq(basenm,sort(basenm))]
cnt1=n_elements(basenm)
cnt2=n_elements(unm)
print,"dups if not equal:",cnt1,cnt2
if (cnt1 ne cnt2) then begin  &$
	iis=sort(basenm) &$
	bnms=basenm[iis] &$
	cntAr=intarr(n_elements(bnms)) &$
	i0=0 &$
	for i=0,cnt1-2 do begin &$
		if bnms[i] eq bnms[i0] then begin &$
			cntar[i0]++ &$
		endif else begin &$
			cntAr[i]=1 &$
			i0=i &$
		endelse &$
	endfor &$
	if bnms[cnt1-1] eq bnms[i0] then begin  &$
		cntar[i0]++ &$
	endif else begin &$
		cntAr[cnt1-1]=1 &$
	endelse &$
	ii=where(cntar ne 1,cnt) &$
	print,"puppi num dups:",cnt &$
;  218 dup basenames
endif
;
uprojid=projid[uniq(projid,sort(projid))]
nproj=n_elements(uprojid)
odir=odirfits + "puppi/"
file_mkdir,odir

for i=0,nproj-1 do begin &$
    ii=where(projid eq uprojid[i],cnt) &$
	iis=sort(basenm[ii])	 &$
    ofile=odir+ uprojid[i]  &$
    openw,lunout,ofile,/get_lun &$
    printf,lunout,":Nfiles:",cnt &$
    printf,lunout,pupnm[ii[iis]],format='(a)' &$
    free_lun,lunout &$
endfor
stop
;--------------------------------------------
; gpu files
;
ii=where(ftar.name eq 'gpu')
iigpu=ftar[ii].ind
ii=where(fitstype eq iigpu,cnt)
gpunm=fitsnm[ii]
; make projid same order as pupnm
projid=strarr(cnt)
basenm=strarr(cnt)
;
; 1 / proj at start
;                 proj / basenm
a=stregex(gpunm,"^([^/]+)/([^/]+\.fits.*)",/sub,/ext)
ii=where(a[2,*] ne '',cnt1)
if cnt1 gt 0 then begin &$
     projid[ii]=reform(a[1,ii]) &$
     basenm[ii]=reform(a[2,ii]) &$
endif
;
; proj 2nd, basenm last
;                        proj    basenm
a=stregex(gpunm,"^[^/]+/([^/]+)/.*/([^/]+\.fits.*)",/sub,/ext)
ii=where((a[2,*] ne '') and (projid eq ''),cnt2)
if cnt2 gt 0 then begin &$
projid[ii]=reform(a[1,ii])  &$
basenm[ii]=reform(a[2,ii])  &$
endif
;
uprojid=projid[uniq(projid,sort(projid))]
nproj=n_elements(uprojid)
odir=odirfits + "gpu/"
file_mkdir,odir

for i=0,nproj-1 do begin &$
    ii=where(projid eq uprojid[i],cnt) &$
    iis=sort(basenm[ii])     &$
    ofile=odir+ uprojid[i]  &$
    openw,lunout,ofile,/get_lun &$
    printf,lunout,":Nfiles:",cnt &$
    printf,lunout,gpunm[ii[iis]],format='(a)' &$
    free_lun,lunout &$
endfor
stop
;--------------------------------------------
; mockfiles 
;
ii=where(ftar.name eq 'mock')
iimock=ftar[ii].ind
ii=where(fitstype eq iimock,cnt)
mocknm=fitsnm[ii]
; make projid same order as mocknm
projid=strarr(cnt)
basenm=strarr(cnt)
;
; proj in basename
; about 10minutes for 4,128,087 entries
; = could speed it up by cutting the subdir..
;
a=stregex(mocknm,"^.*/([^/]+\.fits.*)",/sub,/ext)
ii=where(a[1,*] ne '',cnt)
basenm[ii]=reform(a[1,ii]) 
;
; now get the projid
;
a1=stregex(basenm,"^([^\.]+)",/sub,/ext)
ii=where(a1[1,*] ne '',cnt)
projid[ii]=reform(a1[1,ii])
uprojid=projid[uniq(projid,sort(projid))]
nproj=n_elements(uprojid)
;
odir=odirfits + "mock/"
file_mkdir,odir


for i=0,nproj-1 do begin &$
    ii=where(projid eq uprojid[i],cnt) &$
    iis=sort(basenm[ii])     &$
    ofile=odir+ uprojid[i]  &$
	print,i," "+uprojid[i]+" ",cnt &$
	if uprojid[i] eq 'a2130' then begin &$
	   print,"-->skiping all a2130.. disc size=0. (a2130_dtm,dfrq)ok" &$
	endif else begin &$
    	openw,lunout,ofile,/get_lun &$
    	printf,lunout,":Nfiles:",cnt &$
    	printf,lunout,mocknm[ii[iis]],format='(a)' &$
    	free_lun,lunout &$
	endelse &$
endfor

b1=basenm[ii]
ub1=b1[uniq(b1,sort(b1))]
n1=n_elements(b1)
n2=n_elements(b2)
ndup=n1-n2
print,"dup basenames:",ndup
; 784 dups
stop
;
;-rw-r--r--              0 2020/03/27 05:23:28 MOCKSPEC0/pdevs7/Archived/s3453.20200327.FRBSR26MAR.b6s1g0.00539.fits
;-rw-r--r--              0 2020/03/27 05:23:28 MOCKSPEC0/pdevs7/Archived/s3453.20200327.FRBSR26MAR.b6s1g0.00539.fits
;--------------------------------------------
; planetary..
;
ii=where(ftar.name eq 'planetary')
iiplan=ftar[ii].ind
ii=where(fitstype eq iiplan,cntpl)
plannm=fitsnm[ii]
; make projid same order as pupnm
projid=strarr(cntpl)
basenm=strarr(cntpl)
;
;openw,lunout,'junk.out',/get_lun
;pprintf,lunout,plannm,format='(a)'
;free_lun ,lunout
ok=intarr(cntpl) + 1
a=stregex(plannm,"^[^/]+/([^/]+)/([^/]+)/([^/]+)",/sub,/ext)
a1=reform(a[1,*])
a2=reform(a[2,*])
a3=reform(a[3,*])
;      a1     a2       a3
; xxx/radarn/radardump/proj
; xxx/radarn/movedto/proj
val=strmatch(a2,"radar*")
ii=where(val eq 1,cnt)
if cnt gt 0 then ok[ii]=3
;
val=strmatch(a2,"movedto*")
ii=where(val eq 1,cnt)
if cnt gt 0 then ok[ii]=3
; exclude public_html.. on pat.
ii=where(a3 eq "public_html",cnt)
if cnt gt 0 then ok[ii]=-1
;
val=strmatch(a1 ,"radar*",/fold)
ii=where((val eq 1) and (ok eq 1),cnt)
if cnt gt 0 then ok[ii]=2
ii=where(ok eq 1,cnt)
if cnt gt 0 then ok[ii]=1
;
ii=where(ok eq 3,cnt)
if cnt gt 0 then projid[ii]=a3[ii]
ii=where(ok eq 2,cnt)
if cnt gt 0 then projid[ii]=a2[ii]
ii=where(ok eq 1,cnt)
if cnt gt 0 then projid[ii]=a1[ii]
;
ii=where((projid eq '') and (ok gt 0),cnt)
if cnt gt 0 then projid[ii]=a1[ii]
;
uproj=projid[uniq(projid,sort(projid))]
ii=where(uproj ne '',cnt)
uproj=uproj[ii] 
nproj=n_elements(uproj)
;
odir=odirfits + "planetary/"

file_mkdir,odir

for i=0,nproj-1 do begin &$
    ii=where(projid eq uproj[i],cnt) &$
    ofile=odir+ uproj[i]  &$
    openw,lunout,ofile,/get_lun &$
    printf,lunout,":Nfiles:",cnt &$
    printf,lunout,plannm[ii],format='(a)' &$
    free_lun,lunout &$
endfor
stop

end