From brideout at haystack.mit.edu  Thu Jan  3 15:46:36 2008
From: brideout at haystack.mit.edu (William Rideout)
Date: Thu Jan  3 15:47:46 2008
Subject: [Openmadrigal-admin] Bug fix for updateMaster
Message-ID: <477D49AC.6030701@haystack.mit.edu>

An HTML attachment was scrubbed...
URL: http://www.haystack.mit.edu/pipermail/openmadrigal-admin/attachments/20080103/28a81003/attachment.html
-------------- next part --------------
#!/bin/sh
# The madtclsh path is longer than 32 characters. So, we take advantage
# of the fact that a backslash continues a comment line in tcl \
exec MADROOT/bin/madtclsh "$0" ${1+"$@"}

# $Id: genTab,v 1.7 2008/01/03 14:10:23 brideout Exp $

# usage: genTab

# Loops over all the experiments in the local Madrigal database and
# writes master experiment, file and data tables. The master files,
# located in MADROOT/metadata, are just a concatenation of the
# corresponding files in the individual experiment directories except for
# the experiment ID, which is assigned sequentially in the master files.
# This script is normally invoked by updateMaster.

# modified by B. Rideout 9/21/2005 to overwrite metadata files from text in memory, so that
# there is only a short period of time when Madrigal has incomplete metadata files

proc initializeScanMadFiles {} {
    # Set the glob pattern which determines which experiments are
    # processed.
    global madroot lMadroot siteID
    global fileTabTxt expTabTxt dataTabTxt
    set madroot MADROOT
    set lMadroot [llength [split $madroot /]]
    set siteID SITEID
    set fileTabTxt ""
    set expTabTxt ""
    set dataTabTxt ""
    set expbase $madroot/experiments
    # Process all experiments
    set station *
    # Process all years
    set yearPattern {[12][09][0-9][0-9]}
    # Process all dates
    set datePattern {[0-3][0-9][a-z][a-z][a-z][0-9][0-9]}
    set expPattern $expbase/$yearPattern/$station/$datePattern*/
    return $expPattern
}

proc processExperimentTable {experiment  expID} {
    # Assigns each column of the experiment table in the specified
    # experiment directory to a variable, replaces the experiment ID with
    # expID and appends the row to the master experiment table.
    # Returns 1 if no problems are found, else 0.
    # uses global expTabTxt to hold entire text of new expTab.txt 
    global fileTabTxt expTabTxt dataTabTxt
    set expTabNames [list ExperimentID ExperimentURL ExperimentName SiteID \
                          StartDate StartTime EndDate EndTime \
                          InstrumentCode SecurityCode]
    set expTab expTab.txt
    if {![file exists [file join $experiment $expTab]]} {
        processError "genTab Error: Missing [file join $experiment $expTab]"
        return 0
    }
    set fe [open [file join $experiment $expTab] r]
    gets $fe line
    close $fe
    set expTabEntries [split $line ,]
    set lExpTabEntries [llength $expTabEntries]
    if {$lExpTabEntries != 10} {
        processError "genTab Error: [file join $experiment $expTab] contains $lExpTabEntries entries. There should be 10"
        return 0
    }
    set i 0
    foreach el $expTabEntries {
        incr i
        if {$i == 1} {
            # Experiment ID's are assigned sequentially in output file
	    append expTabTxt "$expID,"
        } elseif {$i == 10} {
            # The remaining fields are unchanged
	    append expTabTxt $el "\n"
        } else {
	    append expTabTxt "$el,"
        }
    }
    return 1
}

proc processFileTable {experiment expID} {
    # Assigns each column of the file table in the specified
    # experiment directory to a variable, replaces the experiment ID with
    # expID and appends the row to the master file table.
    # Returns 1 if no problems are found, else 0.
    # uses global fileTabTxt to hold entire text of new fileTab.txt
    global fileTabTxt expTabTxt dataTabTxt
    set fileTabNames [list FileName ExperimentID DataType Category \
                           Spare1 Spare2 Spare3 AnalysisDate AnalysisTime \
                           Spare4 Spare5]
    set fileTab fileTab.txt
    if {![file exists [file join $experiment $fileTab]]} {
        processError "genTab Warning: Missing [file join $experiment $fileTab]"
        return 0
    }
    set ff [open [file join $experiment $fileTab] r]
    while {[gets $ff line] >= 0} {
        set fileTabEntries [split $line ,]
        set lFileTabEntries [llength $fileTabEntries]
        if {$lFileTabEntries != 11} {
            processError "genTab Error: [file join $experiment $fileTab] contains $lFileTabEntries entries. There should be 11"
            return 0
        }
        set FileName [lindex $fileTabEntries 0]
	set Category [lindex $fileTabEntries 3]
        if {![file exists [file join $experiment $FileName]]} {
	    # this is only an error if not a realtime file
	    if {$Category != 4} {
                processError "genTab Error: Missing [file join $experiment $FileName]"
                return 0
	    }
        }
        set i 0
        foreach el $fileTabEntries {
            incr i
            if {$i == 2} {
                # Experiment ID's are assigned sequentially in output file
		append fileTabTxt "$expID,"
            } elseif {$i == 11} {
                # The remaining fields are unchanged
		append fileTabTxt $el "\n"
            } else {
		append fileTabTxt "$el," 
            }
        }
    }
    close $ff
    return 1
}

proc processDataTable {experiment expID} {
    # Assigns each column of the data table in the specified
    # experiment directory to a variable, replaces the experiment ID with
    # expID and appends the row to the master data table.
    # Returns 1 if no problems are found, else 0.
    # uses global dataTabTxt to hold entire text of new dataTab.txt
    global fileTabTxt expTabTxt dataTabTxt
    set dataTabNames [list Day ExperimentID DataType]
    set dataTab dataTab.txt

    if {![file exists [file join $experiment $dataTab]]} {
        processError "genTab Warning: Missing [file join $experiment $dataTab]"
        return 0
    }
    set fd [open [file join $experiment $dataTab] r]
    while {[gets $fd line] >= 0} {
        set dataTabEntries [split $line ,]
        set lDataTabEntries [llength $dataTabEntries]
        if {$lDataTabEntries != 3} {
            processError "genTab Error: [file join $experiment $dataTab] contains $lDataTabEntries entries. There should be 3"
            return 0
        }
        set i 0
        foreach el $dataTabEntries {
            incr i
            if {$i == 2} {
                # Experiment ID's are assigned sequentially in output file
		append dataTabTxt "$expID,"
            } elseif {$i == 3} {
                # The remaining fields are unchanged
		append dataTabTxt $el "\n"
            } else {
		append dataTabTxt "$el,"
            }
        }
    }
    close $fd
    return 1
}

proc processError {message} {
    # Process errors. In this example, the error message is simply
    # printed.
    puts $message
}

proc getDayno {experiment} {
    # Computes day number yyymmdd from experiment directory name
    # e.g.: /opt/madrigal/experiments/1999/mlh/08oct99/ -> 19990908
    global madroot lMadroot
    global months
    set date [lindex [split $experiment /] [expr $lMadroot+3]]
    set day [string trimleft [string range $date 0 1] 0]
    set month [string range $date 2 4]
    set year [string range $date 5 6]
    scan $year %d yearNum
    if {$yearNum >= 50} {
	set yearNum [expr $yearNum + 1900]
    } elseif {$yearNum < 50} {
	set year [expr $yearNum + 2000]
    }
    for {set m 0} {$m < 12} {incr m} {
	if {$month == [lindex $months $m]} {
	    break
	}
    }
    set dayno [expr 10000*$yearNum + 100*$m + 1*$day]
    return $dayno
}

proc dateCompare {experiment1 experiment2} {
    # Sorting function for lsort
    global madroot lMadroot
    set expDir1 [split $experiment1 /]
    set year1 [lindex $expDir1 [expr $lMadroot+1]]
    set station1 [lindex $expDir1 [expr $lMadroot+2]]
    set expDir2 [split $experiment2 /]
    set year2 [lindex $expDir2 [expr $lMadroot+1]]
    set station2 [lindex $expDir2 [expr $lMadroot+2]]
    set d1 $year1$station1[getDayno $experiment1]
    set d2 $year2$station2[getDayno $experiment2]
    if {$d1 >= $d2} {
        return 1
    } else {
        return -1
    }
}

########################
# Start Execution Here #
########################

# Initialization
set months {jan feb mar apr may jun jul aug sep oct nov dec}
set expPattern [initializeScanMadFiles]

# Get list of experiments and sort by date
set experiments [lsort -increasing -command dateCompare [glob $expPattern]]

set expID [expr 10000000*$siteID]

foreach experiment $experiments {

    incr expID

    # Process the experiment table
    # Every experiment must have an experiment table
    if {![processExperimentTable $experiment $expID]} {
        continue
    }

    # Process the data table
    # Every experiment must have a data table
    if {![processDataTable $experiment $expID]} {
        continue
    }

    # Process the file table
    # Experiments do not have to have a file table
    processFileTable $experiment $expID

}


set feout [open [file join $madroot metadata expTab.txt] w]
set ffout [open [file join $madroot metadata fileTab.txt] w]
set fdout [open [file join $madroot metadata dataTab.txt] w]
puts -nonewline $feout $expTabTxt
puts -nonewline $ffout $fileTabTxt
puts -nonewline $fdout $dataTabTxt
close $feout
close $ffout
close $fdout

catch {file attributes [file join $madroot metadata expTab.txt] -permissions g+w}
catch {file attributes [file join $madroot metadata fileTab.txt] -permissions g+w}
catch {file attributes [file join $madroot metadata dataTab.txt] -permissions g+w}

exit