The metadata.tcl Script

Modification Date: 11/26/2008

Table of Procedures

${API}::atExit metadata metadata::cmdTime metadata::destructObjs metadata::destructThreads metadata::evalCmd metadata::init

metadata::killJob metadata::log metadata::registerObj metadata::runningInsertStatsFile metadata::setSchema metadata::submit metadata::wakeupDataRecvThreads

metadata.tcl Version 1.0 Wraps the metadataAPI.so and the genericAPI.so and the genericAPI.tcl for use by the metadata API.
Provides data retrieval functions for the LDAS.
Release Date: Not Available The caller wishing to use these functions without qualifiying the names with metadata:: should include the line:
namespace import metadata::*.
after requiring or sourcing this code. Possibly qualfying this with:
namespace forget metadata::someProc.
Which will make the forgotten commands inaccesible outside of the metadata namespace. (rendering them "private" in the c++ class sense.)
Run environment: requires DB2 client side:
ODBC driver DM: IBM Driver Manager Dataset name: ldas_tst All data for job is saved in global ::${jobid} array

package provide metadata 1.0
package require metadataAPI
package require metadataSpectrum
package require metaCallback
package require metaschema
package require getMetaData
package require putMetaData
namespace eval metadata {}

proc ${API}::atExit {} {
	;## remove the threads when cancel threads work
    set threads [ getThreadList ]
    set text "cleaning up threads $threads"
    foreach { tid func status } [ getThreadList ] {
        regexp {\(([^\)]+)\)} $func -> threadfunc
        append text "thread function ${threadfunc}_r"
        ${threadfunc}_r $tid
    }
	set objects [ getElementList ]
    append text "Threads $threads "
	append text "destroying objects $objects, "
	foreach datap $objects {
		destructElement $datap
	}
	append text "channels [ countChannels ]"	
    addLogEntry $text
}

• msg msg to log

 metadata::log $msg

proc metadata::log { { msg "" } { errlvl 0 } { caller "" } } {
     catch { set jobid [ uplevel set jobid ] }
     set log $::LOCAL_LOG_FILE
     if { [ catch {
        if { ! [ string compare "" $caller ] } {
           set caller [ uplevel myName ]
        }
        if  { $errlvl || $::DEBUG > 1 } {
    	    addLogEntry "'$msg'" $errlvl $caller [ gpsTime ] $log
        }
     } err ] } {
       if { $::DEBUG } {
          puts stderr "[ myName ]: $err"
       }
     }
}

• None.
  

metadata::Init
Returns:


None.

proc metadata::init {} {
    if  { [ catch {
        ;## patterns for retry
        set patfile [ file join $::TOPDIR metadataAPI SQLretries.putMetaData ]
	    if	{ ! [ file exist $patfile ] } {
		    set patfile [ file join $::LDAS lib metadataAPI SQLretries.putMetaData ]
	    }
	    set patterns [ dumpFile $patfile ]
        set patterns [ split $patterns \n ]
        set temp [ list ]
   	    foreach pattern $patterns {
		    if	{ [ regexp {^#} $pattern ] } {
			    continue
		    }
            regexp {([^#\s\t]+)} $pattern -> pattern
            lappend temp "($pattern)"
        }
        set ::pattern_putMetaData [ join $temp \| ]
        set patfile [ file join $::TOPDIR metadataAPI SQLretries.getMetaData ]
	    if	{ ! [ file exist $patfile ] } {
		    set patfile [ file join $::LDAS lib metadataAPI SQLretries.getMetaData ]
	    }
	    set patterns [ dumpFile $patfile ]
        set patterns [ split $patterns \n ]
        set temp [ list ]
   	    foreach pattern $patterns {
		    if	{ [ regexp {^#} $pattern ] } {
			    continue
		    }
            regexp {([^#\s\t]+)} $pattern -> pattern
            lappend temp "($pattern)"
        }
        set ::pattern_getMetaData [ join $temp \| ]
        ;## set to 2 to get one retry
        if  { ! [ info exist ::MAX_TRIES ] } {
            set ::MAX_TRIES 2
        }
        if  { ! [ info exist ::METADATA_MAX_JOB_TIME ] } {
            set ::METADATA_MAX_JOB_TIME 50000
        }
        if  { ! [ info exist ::THREAD_TIMEOUT_SECS ] } {
            set  ::THREAD_TIMEOUT_SECS  50
        }
        if	{ [ regexp {8.4} $::tcl_version ] } {
    		set ::metadata::tcl8.4 1
        }
        ;## bgLoop metadata_wakeupDataRecvThreads metadata::wakeupDataRecvThreads 60
    } err ] } {
        addLogEntry $err 2
    }
}

• jobid - original jobid
• cmd - cmd to eval
• cmdstem - e.g. metadataSlave

execSlaveCmd $slave
Returns:


results of cmd

proc metadata::cmdTime { jobid } {
     if	{ [ catch {
        if  { [ info exist ::${jobid}(cmdstem) ] } {
            set cmdstem [ set ::${jobid}(cmdstem) ]
            set dbname  [ set ::${jobid}(-database) ]
		    ;## usage log for queries
            if  { [ regexp {getMetaData|descMetaData} $cmdstem ] } {
                set wallsecs [ __t::mark $jobid ]
                __t::cancel $jobid
			    set begintic [ set ::${jobid}(begintic) ]
			    # set endtic [ exectime ]
                set endtic 1.0
			    set rows [ set ::${jobid}(Rows) ]
			    if	{ ! [ info exist ::${dbname}(queries) ] } {
				    set ::${dbname}(queries) 0
			    }
                incr ::${dbname}(queries) $rows
			    addLogEntry "$cmdstem for database $dbname CPUtime=[ expr $endtic - $begintic ] secs, \
			    walltime=$wallsecs secs; received [set ::${dbname}(queries)] queries since start up." blue
            } else {
			    set dims [ set ::${jobid}(dims) ]
			    if	{ ! [ info exist ::${dbname}(inserted) ] } {
				    set ::${dbname}(inserted) 0
			    }
                incr ::${dbname}(inserted) $dims
			    ;## timing already done in slave
			    addLogEntry "$cmdstem inserted [set ::${dbname}(inserted)] rows into database $dbname since start up." blue
            }
        }
    } err ] } {
        return -code error $err
   	}
}

• None.
  

cleanupJob
Returns:


None.

proc metadata::cleanupJob { jobid caller { abort 0 } } {
    set jobexist 0
	if	{ [ catch {
        if  { [ info exist ::${jobid} ] } {
            set jobexist 1
            if  { ! [ info exist ::${jobid}(errors) ] && ! $abort } {
                metadata::cmdTime $jobid
            }
		}
    } err ] } {
        log  $err 2
	}
    # catch { destructObjs $jobid }
    ;## may overwrite a reused thread
    # catch { destructThreads $jobid }
	if	{ [ catch {
        if  { $jobexist } {
            set callback [ set ::${jobid}(cmd) ]
		    eval $callback $jobid 	
        }
	} err ] } {
		addLogEntry "$jobid called by $caller: $err" blue
	}	
}

• cmd

metadata::evalCmd $cmd
Returns:


None.

proc metadata::evalCmd { jobid cmd } {
	set seqpt ""
	set dbname $::DATABASE_NAME
	if	{ [ catch {
		set cid [ set ::${jobid}(cid) ]		
		if	{ ! [ regexp -- {-database\s+[\{]*(\S+)[\}]*\s+} $cmd -> dbname ] } {
			set dbname $::DATABASE_NAME
		}
		set seqpt "check $dbname"
    	regsub -all {\{} $dbname {} dbname
    	regsub -all {\}} $dbname {} dbname
    	if  { ! [ string length $dbname ] } {
        	set dbname $::DATABASE_NAME
    	}
		set ::${jobid}(errlvl) 0
        set ::${jobid}(-database) $dbname
    	if  {  [ regexp "^\[ \t]*$"  $cmd ] } {
        	set ::${jobid}(errlvl) 3
        	log "malformed command: cmd=$cmd,$errmsg"
        	set result "malformed command: cmd=$cmd,$errmsg"
        	error $result
    	}
    	set cmdstem [ lindex $cmd 0 ]
		set callback [ set ::${jobid}(cmd) ]
        set ::${jobid}(cmdstem) $cmdstem
        ;## validate the database
        ;## IF THE DSO IN A DATAPIPELINE OR DATASTANDALONE/PUTSTANDALONE JOB IS NOT
        ;## IN THE /LAL/LIBRARY DIRECTORY (THE OFFICIAL LAL RELEASE DIRECTORY) THEN
        ;## LDAS MUST OVERWRITE AND FORCE THE RESULTS TO GO IN THE TEST DATABASE.
        if  { [ info exist ::${jobid}(-dynlib) ] } {
            set dso [ set ::${jobid}(-dynlib) ]
            if  { ! [ regexp {^lib} $dso ] &&
              ! [ regexp $::DYNLIB_DIRECTORY $dso ] &&
              ! [ regexp {(tst|test)} $dbname ] } {
                error "non standard DSO $dso should be inserted \
                into test database only, "
            }
        }
        if  { ! [ info exist ::${dbname}(login) ] } {
                error "$dbname is not a valid database name, \
                must be [ set ::site_dbnames($::LDAS_SYSTEM) ]"
        }
        eval $cmdstem $jobid			
	} err ] } {
        metadata::putMetaDataOjDel $jobid
		set msg "cmd=$cmd, database $dbname, $err"
        set ::${jobid}(errors) $err
        if  { [ regexp -nocase {mysterious} $err ] } {
            log "$seqpt: $msg" 3
        } else {
		    log "$seqpt: $msg" 2
        }
        cleanupJob $jobid [ myName ]
	}
}

• cmd

metadata::killJob $jobid
Returns:


None.

proc ${API}::killJob { jobid } {
    if  { [ catch {
        cleanupJob $jobid [ myName ] 1
    } err ] } {
        log $err 2
    }
}

proc metadata::runningStatsFile {  jobid cmd rows { table "" } } {
     if { [ catch {
	 		
       	set endtime [ gpsTime ]
        if	{ [ info exist ::${jobid}(-userid) ] } {
			set userid [ set ::${jobid}(-userid) ]
		} else {
			set userid ldas
		}
        set dbname [ set ::${jobid}(-database) ]
        set usercmd [ set ::${jobid}(-commandname) ]
        if  { [ regexp {dataPipeline} $usercmd ] } {
            set dso [ set ::${jobid}(-dynlib) ]
            set usercmd "${usercmd}($dso)"
        }
        set active  $::LDASLOG/$cmd.log
        set archive $::LDASARC/jobstats_archive/$cmd.log.$endtime
        set dir $::LDASARC/jobstats_archive
		
        if { ! [ file isdirectory $dir ] } {
           file mkdir $dir
           file attributes $dir -permissions 04755
           gifBalls $dir
        }
        set mb [ expr { 1024 * 1024 } ]
        if { [ file exists $active ] } {
           if { [ file size $active ] >= $mb } {
              file rename $active $archive
           }
        }
		set fid   [ open $active a+ ]
        puts $fid "$endtime $userid $usercmd $dbname $rows $table"
        ::close $fid
		
        file attributes $active -permissions 0644
     } err ] } {
        catch { ::close $fid }
        return -code error "[ myName ]: $err"
     }
}

• registerObj name of ilwd object

 registerStmt $stmtp

proc metadata::registerObj { jobid elemp } {
	if	{ [ array exist ::${jobid} ] } {
    	lappend ::${jobid}(objects) $elemp
        debugPuts "registered $elemp"
	}
}

• None.
  

 destructObjs

proc metadata::destructObjs { jobid } {
	if	{ [ catch {
	if	{ [ info exist ::${jobid}(objects) ] } {
    	foreach elemp [ set ::${jobid}(objects) ] {
        	if { [ catch {
                destructElement $elemp
                if  { [ string length [ info commands $elemp ] ] } {
                    debugPuts "rename $elemp"
                    rename $elemp {}
                }
			} err ] } {
				log "destructElement $elemp error: $err" 2
			}
		}
		unset ::${jobid}(objects)
    }
	} err ] } {
		log $err 2
		return -code error "[myName], $err"
	}
}

• dbquery SQL statement

       metadata::submit dbquery

proc metadata::submit { { dbquery "" } { readonly 0 } } {
    if    { $dbquery == "" } {
        return -code error "metadata::submit query cannot be empty."
    }
    set dbquery [string trim $dbquery "\"\}\{"]
#   create a statement
    set stmtp [ createStmt $readonly ]
	set ::${::jobid}(stmtp) $stmtp
#   submit the statement, ignore DROP statement errors if table does
#   not already exist
#
    if    { [ catch {
          dbEasyStmt_Submit $stmtp $dbquery
    } errmsg ] } {
          log "Statement Submit error: $errmsg" 2
          if    {  ! [regexp -nocase -- {^-|drop|^[ \t]+$} $dbquery] } {
                return -code error $errmsg
          }
    }
    return -code ok $stmtp
}

 metadata::setSchema $stmtp

proc metadata::setSchema { stmtp dbuser } {
	if	{ ! [ string compare $::DBSCHEMA "IBMdb2" ] } {
		if	{ [ catch {
			set sqlcmd "set schema $dbuser"
			dbEasyStmt_Prepare $stmtp $sqlcmd
			dbEasyStmt_Submit $stmtp
			log "submitted set schema for $dbuser"
		} err ] } {
			return -code error $err
		}	
	}
}

 destructThreads $jobid

proc metadata::destructThreads { jobid } {
    if  { [ catch {
     	foreach var [ info vars ::metadata::threads_* ] {
        	# addLogEntry "var $var, [ set $var ] jobid $jobid" purple
            set thread_jobid [ set $var ]
        	if	{ [ regexp {metadata::threads_(\S+)} $var -> tid ] } {
            	if	{ [ string equal $jobid $thread_jobid ] } {
            		set threadfunc none
            		catch { set threadfunc [ getThreadFunction $tid ] }
            		addLogEntry "cleaning up $thread_jobid $tid '$threadfunc'" purple
                    if	{ ! [ string equal none $threadfunc ] } {
            			catch { ${threadfunc}_r $tid }
                    }
                }
        	}
        }
    } err ] } {
    	addLogEntry $err red
    }
}

proc metadata::wakeupDataRecvThreads  {} {
    if  { [ array exist ::dataRecvStates ] } {
        foreach tid [ array names ::dataRecvStates ] {
            foreach { sid currtime } [ set ::dataRecvStates($tid) ] { break }
            dataRecvReaper $tid $sid
        }
    }
}