Previous: Internal Madrigal Python API   Up: Internal Madrigal Python API   Next: madrigal.cedar

Top

madrigal.admin module

The admin module contains all administrative classes relating to the madrigal python api.

The main role of this module is to update the data in the Madrigal database. Also contains a notification class and a standard error handing class.

$Id: admin.py 7044 2019-10-07 19:13:16Z brideout $

"""The admin module contains all administrative classes relating to the madrigal python api.

The main role of this module is to update the data in the Madrigal database.  Also contains a
notification class and a standard error handing class.

$Id: admin.py 7044 2019-10-07 19:13:16Z brideout $
"""
# standard python imports
import os, os.path, sys
import smtplib
import datetime, time
import types
import traceback
import shutil
import hashlib
import urllib
import re
import glob
import warnings
import subprocess
import hashlib

# third party inports
import numpy

# Madrigal imports
import madrigal.metadata
import madrigal.openmadrigal
import madrigal.data
import madrigal.cedar
import madrigal.ui.userData

def convertMad2FileToMad3(mad2File, madDB=None, status=None):
    """convertMad2FileToMad3 will convert an input mad2File to Madrigal 3 HDF5.  It returns the path to the
    newly creates file, which will be in the /tmp directory.
    
    Inputs:
    
        mad2File - A Madrigal 2 format file to convert
        
        madDB - a madrigal.metadata.MadrigalDB object.  If None (the default) created.
        
        status - if None, get status from fileTab.txt.  Else use status passed in
        
    Returns: full path to newly created Madrigal 3 Hdf5 file.  Basename is basename of input file, with .hdf5 extension
    """
    if madDB is None:
        madDB = madrigal.metadata.MadrigalDB()
        
    # make sure an hdf5 file not passed in
    base, ext = os.path.splitext(mad2File)
    if ext in ('.hdf5', '.h5', '.hdf5'):
        raise ValueError('Cannot call convertMad2FileToMad3 with Hdf5 file %s' % (mad2File))
        
    madDataObj = madrigal.data.MadrigalFile(mad2File, madDB)
    kinst = madDataObj.getKinstList()[0]
    kindat = madDataObj.getKindatList()[0]
    
    extraParms, indParms, splitParms = madDB.getKinstKindatConfig(kinst, kindat)
    
    tmpFile = os.path.join('/tmp', os.path.basename(mad2File) + '.hdf5')
    # make sure it doesn't exist
    if os.access(tmpFile, os.R_OK):
        os.remove(tmpFile)
    
    madDataObj.exportToHdf(tmpFile, indParms, splitParms, extraParms, status=status)
    
    return(tmpFile)
    
    


class MadrigalDBAdmin:
    """MadrigalDBAdmin is a class that allows modifications to be made to the Madrigal database

    dbAdminObj = madrigal.admin.MadrigalDBAdmin()

    expDir = dbAdminObj.createMadrigalExperiment('/home/hyperion/brideout/mlh050429c.000',
                                        'Dummy experiment',
                                        0,
                                        'test exp',
                                        30,
                                        1)

    Non-standard Python modules used: None

    Change history:

    Written by "Bill Rideout":mailto:wrideout@haystack.mit.edu  May. 5, 2005

    """
    
    def __init__(self, madDB = None):
        """__init__ initializes MadrigalDBAdmin
        
        Inputs: madDB - Existing MadrigalDB object.  Default = None.
        
        Returns: void

        Affects:

            Sets self.__madDB to MadrigalDB object
            Sets self.__madInst to MadrigalInstrument object
        """

        if madDB == None:
            self.__madDB = madrigal.metadata.MadrigalDB()
        else:
            self.__madDB = madDB

        self.__madInst = madrigal.metadata.MadrigalInstrument(self.__madDB)
        self.__openMad = madrigal.openmadrigal.OpenMadrigal(self.__madDB)
        self.__madSite = madrigal.metadata.MadrigalSite(self.__madDB)
        self._userData = madrigal.ui.userData.MadrigalUserData(self.__madDB)
        
            

    def createRTExperiment(self,
                           startTime,
                           numDays,
                           instrument,
                           expTitle,
                           rtFilenameList,
                           kindatList,
                           permissionList,
                           fileDescList,
                           optChar = '',
                           endTime = None,
                           security = 0,
                           dirName = None,
                           experimentsDirNum=None,
                           PI='', PIEmail='', 
                           fileAnalystList=None, fileAnalystEmailList=None,
                           notify = True):
        """createRTExperiment creates a new experiment on Madrigal in preparation for realtime data.

        Since the experiment is presumably not yet complete, metadata such as the duration of the experiment
        must be estimated.  This metadata will be overwritten when the first batch file is added.

        Inputs:
        
            startTime - experiment start time.  If a number, assumed to be seconds since 1/1/1970.  May also
            be a datetime.datetime object
            
            numDays - number of days the experiment is estimated to run.  Ignored if optional endTime given.
            
            instrument - instrument code or 3-letter Madrigal mnenonic
            
            expTitle - experiment title
            
            rtFilenameList - list of realtime filenames to be created. Must all be Hdf5 files.
            
            kindatList - list of ints of kindats for each realtime file.  Len = len(rtFilenameList)
            
            permissionList - list of 0 (public) or 1 (private). Len = len(rtFilenameList)
            
            fileDescList - list of realtime file descriptions

            optChar - optional character to be added to experiment directory if no dirName
                      given.  If dirName argument given, this argument ignored.  optChar
                      is used if the default directory name DDmmmYY is used for
                      more than one experiment created for a given instrument on a given day.
                      For example, if --optChar=h for a MLH experiment on September 12, 2005,
                      then the experiment directory created would be experiments/2005/mlh/12sep05h.

            endTime - optional end date and time of experiment.  If a number, assumed to be seconds since
            1/1/1970.  May also be a datetime.datetime object

            security - experiment security setting.  If 0 (the default) public.  If 1, private.
                       If -1, entire experiment ignored.  Any given file permission is the more
                       restricted of experiment permission and file permission.

            dirName - directory name to use for experiment.  If None (the default), the directory
                      name will be the default name DDmmmYY[optChar].  Cannot contain "/"
                      
            experimentsDirNum - the number to be appended to the experiments directory, if experiments
                      directory being used is of the form experiments[0-9]* instead of just
                      experiments.  For example, if experimentsDirNum is 7, then the experiment
                      would be created in MADROOT/experiments7 instead of MADROOT/experiments.
                      
            PI- full name of principal investigator.  The default is ''
            
            PIEmail - email of principal investigator.  The default is ''
            
            fileAnalystList - list of full names of file analysts, one for each file.  If None, the default, 
                File Analyst = ''
            
            fileAnalystEmailList - list of emails of file analysts, one for each file.  If None, the default, 
                File Analyst email = ''
                
            notify - if True (the default), send a message to all registered users.  If False, do not.

        Returns:

            Full path to directory created
            
        """
        # check optChar
        if type(optChar) not in (bytes, str):
            raise ValueError('optChar must be an empty or a one character string, not %s' % (str(optChar)))

        if len(optChar) > 1:
            raise ValueError('optChar must be an empty or a one character string, not %s' % (str(optChar)))

        security = int(security)
        if security not in (-1,0,1):
            raise ValueError('security must be -1, 0, or 1, not %i' % (security))
        
        # convert startTime to datetime if needed
        if type(startTime) in (int, int, float):
            startTime = datetime.datetime.utcfromtimestamp(startTime)

        # create endTime based on numDays
        if endTime != None:
            if type(endTime) in (int, int, float):
                endTime = datetime.datetime.utcfromtimestamp(startTime)
        else:
            if numDays >= 0:
                endTime = startTime + datetime.timedelta(numDays) - datetime.timedelta(0,1)
            else:
                raise ValueError('numDays must not be negative')

        if startTime >= endTime:
            raise ValueError('Experiment start time %s after end time %s' % (str(startTime),
                                                                               str(endTime)))
        

        # get instrument mnemonic and instCode
        try:
            instCode = int(instrument)
            instMnemonic = self.__madInst.getInstrumentMnemonic(instCode)
        except ValueError:
            if len(instrument) != 3:
                raise ValueError('%s not a legal instrument mnemonic' % (str(instrument)))
            instMnemonic = instrument.lower()
            # verify its a legal mnemonic
            instList = self.__madInst.getInstrumentList()
            found = False
            for inst in instList:
                if instMnemonic == inst[1]:
                    found = True
                    instCode = inst[2]
                    break
            if found == False:
                raise ValueError('%s not a legal instrument mnemonic or code' % (str(instrument)))

        if instMnemonic == None:
            raise ValueError('%s not a legal instrument mnemonic or code' % (str(instrument)))
        
        instDesc = self.__madInst.getInstrumentName(instCode)

        # expTitle
        if type(expTitle) not in (bytes, str) and expTitle != None:
            raise ValueError('expTitle not a string')
        if expTitle == None:
            expTitle = ''
        if expTitle.find(',') != -1:
            raise ValueError('expTitle cannot contain a comma')
            
        # PI
        if type(PI) not in (bytes, str) and PI != None:
            raise ValueError('PI not a string')
        if PI == None:
            PI = ''
        if PI.find(',') != -1:
            raise ValueError('PI cannot contain a comma')
            
        # PIEmail
        if type(PIEmail) not in (bytes, str) and PIEmail != None:
            raise ValueError('PIEmail not a string')
        if PIEmail == None:
            PIEmail = ''
        if PIEmail.find(',') != -1:
            raise ValueError('PIEmail cannot contain a comma')

        # rtFilenameList
        if type(rtFilenameList) not in (list, tuple):
            raise ValueError('rtFilenameList not a list or tuple')
        # make sure each rtFilename is a string without /
        for filename in rtFilenameList:
            if type(filename) not in (bytes, str):
                raise ValueError('rtFilenameList must contain strings')
            if filename.find('/') != -1:
                raise ValueError('rtFilenameList must contain strings without /')
            if filename.find(',') != -1:
                raise ValueError('filename cannot contain a comma')
            base, ext = os.path.splitext(filename)
            if ext not in ('.hdf5', '.h5', '.hdf5'):
                raise ValueError('All input files must have a valid Hdf5 extension, <%s> does not' % (filename))

        # kindatList
        if len(kindatList) != len(rtFilenameList):
            raise ValueError('length of kindatList not equal length of rtFilenameList')
        for item in kindatList:
            try:
                int(item)
            except:
                raise ValueError('kindatList must contain integers')

        # permissionList
        if len(permissionList) != len(rtFilenameList):
            raise ValueError('length of permissionList not equal length of rtFilenameList')
        for item in permissionList:
            try:
                permission = int(item)
            except:
                raise ValueError('permissionList must contain integers')

            if permission not in (0,1):
                raise ValueError('permissionList must contain integers of value 0 (public) or 1 (private)')

        
        # fileDescList
        if len(fileDescList) != len(rtFilenameList):
            raise ValueError('length of fileDescList not equal length of rtFilenameList')
        for item in fileDescList:
            if type(item) not in (bytes, str):
                raise ValueError('fileDescList must only contain strings')
            if item.find(',') != -1:
                raise ValueError('fileDesc cannot contain a comma')
            
        # fileAnalystList
        if fileAnalystList:
            if len(fileAnalystList) != len(rtFilenameList):
                raise ValueError('length of fileAnalystList not equal length of rtFilenameList')
            for item in fileAnalystList:
                if type(item) not in (bytes, str):
                    raise ValueError('fileAnalystList must only contain strings')
                if item.find(',') != -1:
                    raise ValueError('fileAnalyst cannot contain a comma')
                
        # fileAnalystEmailList
        if fileAnalystEmailList:
            if len(fileAnalystEmailList) != len(rtFilenameList):
                raise ValueError('length of fileAnalystEmailList not equal length of rtFilenameList')
            for item in fileAnalystEmailList:
                if type(item) not in (bytes, str):
                    raise ValueError('fileAnalystEmailList must only contain strings')
                if item.find(',') != -1:
                    raise ValueError('fileAnalystEmail cannot contain a comma')
            
        # experimentDirNum
        if experimentsDirNum != None:
            experimentsDir = 'experiments%i' % (experimentsDirNum)
            # verify this directory exists
            if not os.access(os.path.join(self.__madDB.getMadroot(), experimentsDir), os.R_OK):
                raise ValueError('no such directory %s' % (experimentsDir))
        else:
            experimentsDir = 'experiments'

        # all the arguments check out - create the directory
        # create experiment dir
        expDir2 = os.path.join('%i' % (startTime.year),
                               instMnemonic)
        
        if dirName == None:
            dirName = startTime.strftime('%d%b%y').lower() + optChar
        else:
            # verify dirName is basename
            if dirName.find('/') != -1:
                raise ValueError('dirName must be base directory name, not %s' % (dirName))

        expDir2 = os.path.join(expDir2, dirName)

        expDir = os.path.join(self.__madDB.getMadroot(), experimentsDir, expDir2)
        
        expDir3 = os.path.join(experimentsDir, expDir2)

        # if the directory already exists, raise error
        if os.access(expDir, os.R_OK):
            raise IOError('Directory %s already exists' % (expDir))

        os.makedirs(expDir)
        os.chmod(expDir, 0o775)

        # expTab.txt
        expTabText = '0,' + self.__madDB.getTopLevelUrl()
        if expTabText[-1] != '/':
            expTabText += '/madtoc/'
        else:
            expTabText += 'madtoc/'
        expTabText += '%s,%s,%i,%04i%02i%02i,%02i%02i%02i,%04i%02i%02i,%02i%02i%02i,%i,%i,%s,%s\n' %(expDir3,
                                                                                                     expTitle,
                                                                                                     self.__madDB.getSiteID(),
                                                                                                     startTime.year,
                                                                                                     startTime.month,
                                                                                                     startTime.day,
                                                                                                     startTime.hour,
                                                                                                     startTime.minute,
                                                                                                     startTime.second,
                                                                                                     endTime.year,
                                                                                                     endTime.month,
                                                                                                     endTime.day,
                                                                                                     endTime.hour,
                                                                                                     endTime.minute,
                                                                                                     endTime.second,
                                                                                                     instCode,
                                                                                                     security,
                                                                                                     PI,
                                                                                                     PIEmail)

        # write expTab.txt
        f = open(os.path.join(expDir, 'expTab.txt'), 'w', encoding='utf-8')
        f.write(expTabText)
        f.close()
        os.chmod(os.path.join(expDir, 'expTab.txt'), 0o664)

        # fileTab.txt
        # set file times to right now UT
        nowUT = datetime.datetime.utcnow()
        nowDate = nowUT.strftime('%Y%m%d')
        nowTime = nowUT.strftime('%H%M%S')
        fileTabText = ''
        for index in range(len(rtFilenameList)):
            
            if fileAnalystList and fileAnalystEmailList:
                fileAnalyst = fileAnalystList[index]
                fileAnalystEmail = fileAnalystEmailList[index]
            else:
                fileAnalyst = ''
                fileAnalystEmail = ''
            
            fileTabText += rtFilenameList[index]
            fileTabText += ',0,%i,4,0,0,0,%s,%s,' % (kindatList[index], nowDate, nowTime)
            fileTabText += '%s,%i,%s,%s\n' % (fileDescList[index], permissionList[index],
                                              fileAnalyst, fileAnalystEmail)

        # write fileTab.txt
        f = open(os.path.join(expDir, 'fileTab.txt'), 'w')
        f.write(fileTabText)
        f.close()
        os.chmod(os.path.join(expDir, 'fileTab.txt'), 0o664)

        # create all writeable directory overview
        os.makedirs(os.path.join(expDir, 'overview'))
        os.chmod(os.path.join(expDir, 'overview'), 0o777)
        
        if notify:
            # get expPath without MAD ROOT
            expPath = expDir[expDir.find('experiments'):]
            if expPath[-1] == '/':
                expPath = expPath[:-1]
            madNotify = MadrigalNotify(self.__madDB)
            message = 'You requested to be notified when new data from instrument %s was available. A new realtime experiment for that instrument has been created at %s at %s. '   \
                % (instDesc, self.__madDB.getTopLevelUrl(), str(datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S')))
            message += 'Please contact %s if you wish to be unregistered.' %  (self.__madDB.getContactEmail())
                
            # deal with users registered for instrument in general
            userList = self._userData.getRegisteredInstUsers(instCode)
            if len(userList) > 0:
                for user in userList:
                    try:
                        madNotify.notify(user, message, 'Update to Madrigal instrument you registered interest in')
                    except:
                        print(('Warning - email to %s failed to be sent' % (user)))

        return(expDir)
        
        
    def writeRTMadrigalFile(self,
                            expDir,
                            rtFilename,
                            rtFile):
        """writeRTMadrigalFile writes a realtime Madrigal file to a Madrigal experiment directory.

        Fails if rtFilename does not match one listed in fileTab.txt.

        Inputs:
        
            expDir - full path to experiment directory (as returned by createRTExperiment)
            
            rtFilename - basename of realtime file to be writtem
            
            rtFile - a string containing the realtime file contents

        Returns: None

        Raises exception if rtFilename does not match one listed in fileTab.txt.
        """
        # verify rtFilename listed in fileTab.txt
        try:
            fileInfo = madrigal.metadata.MadrigalMetaFile(self.__madDB, os.path.join(expDir, 'fileTab.txt'))
        except:
            raise ValueError('Unable to open fileTab.txt in %s' % (expDir))
        
        if fileInfo.getHasCatalogByFilename(rtFilename) == None:
            raise ValueError('Filename %s not found in fileTab.txt' % (rtFilename))

        # okay - write it
        f = open(os.path.join(expDir, rtFilename), 'w', encoding='utf-8')
        f.write(rtFile)
        f.close()
        os.chmod(os.path.join(expDir, rtFilename), 0o664)


    def appendRTMadrigalFile(self,
                             expDir,
                             rtFilename,
                             rtFile):
        """appendRTMadrigalFile used to allow appending to a realtime Madrigal file. No longer possible with Hdf5 format.

        Inputs:
        
            expDir - full path to experiment directory (as returned by createRTExperiment)
            
            rtFilename - basename of realtime file to be writtem
            
            rtFile - a string containing the new realtime file contents

        Always raises IOError
        """
        raise IOError('The method appendRTMadrigalFile no longer supported by Madrigal 3.0')
        


    def createMadrigalExperiment(self,
                                 madFilename,
                                 expTitle,
                                 permission,
                                 fileDesc,
                                 instCode = None,
                                 category = 1,
                                 optChar = '',
                                 dirName = None,
                                 kindat = None,
                                 experimentsDirNum=None,
                                 PI='', PIEmail='', 
                                 fileAnalyst='', fileAnalystEmail='',
                                 createCachedText=False, createCachedNetCDF4=False,
                                 notify = True, updateToMad3=False):
        """createMadrigalExperiment creates a new experiment on Madrigal using metadata read from madFilename.

        Inputs:
        
            madFilename - full path to the complete Madrigal file.  Basename will be maintained.
            
            expTitle - experiment title
            
            permission - 0 (public) or 1 (private) or -1 (ignore). 
            
            fileDesc - file description

            instCode - instrument code.  If default (None), instrument code is taken from file, but error
            is thrown if more than one kinst found.

            category - 1=default, 2=variant, 3=history, or 4=realtime. Default is 1 (default file)

            optChar - optional character to be added to experiment directory if no dirName
                      given.  If dirName argument given, this argument ignored.  optChar
                      is used if the default directory name DDmmmYY is used for
                      more than one experiment created for a given instrument on a given day.
                      For example, if --optChar=h for a MLH experiment on September 12, 2005,
                      then the experiment directory created would be experiments/2005/mlh/12sep05h.

            dirName - directory name to use for experiment.  If None (the default), the directory
                      name will be the default name DDmmmYY[optChar].  Cannot contain "/"

            kindat - if not None (the default), use this kindat instead of what is found in the file.
            
            experimentsDirNum - the number to be appended to the experiments directory, if experiments
                      directory being used is of the form experiments[0-9]* instead of just
                      experiments.  For example, if experimentsDirNum is 7, then the experiment
                      would be created in MADROOT/experiments7 instead of MADROOT/experiments.
                      
            PI- full name of principal investigator.  The default is ''
            
            PIEmail - email of principal investigator.  The default is ''
            
            fileAnalyst -full name of file analyst.  The default is ''
            
            fileAnalystEmail - email of file analyst,.  The default is ''
            
            createCachedText - if True, add cached text file in overview/.txt.gz.  If False,
                no cached file.
                
            createCachedNetCDF4 - if True, add cached netCDF4 file in overview/.nc.  If False,
                no cached file.
                
            notify - if True (the default), send a message to all registered users.  If False, do not.
            
            updateToMad3 - if False (the default), error raised if madFilename non-Hdf5 file. If True, try to
                convert madFilename to Madrigal with .hdf5 extension before loading.
            
        Returns:

            Full path to directory created
            
        """
        # check optChar
        if type(optChar) not in (bytes,str):
            raise ValueError('optChar must be an empty or a one character string, not %s' % (str(optChar)))

        if len(optChar) > 1:
            raise ValueError('optChar must be an empty or a one character string, not %s' % (str(optChar)))
        
        base, ext = os.path.splitext(madFilename)
        if updateToMad3:
            if ext not in ('.hdf5', '.h5', '.hdf5'):
                madFilename = convertMad2FileToMad3(madFilename, self.__madDB, fileDesc)
        elif ext not in ('.hdf5', '.h5', '.hdf5'):
            raise ValueError('called createMadrigalExperiment with non-Hdf5 file <%s> and updateToMad3 False' \
                % (madFilename))
                
        fileInfo = madrigal.data.MadrigalFile(madFilename, self.__madDB)

        # get startTime
        sTime = fileInfo.getEarliestTime()
        startTime = datetime.datetime(sTime[0],sTime[1],sTime[2],sTime[3],sTime[4],sTime[5])

        # get endTime
        eTime = fileInfo.getLatestTime()
        endTime = datetime.datetime(eTime[0],eTime[1],eTime[2],eTime[3],eTime[4],eTime[5])
        
        # get instrument mnemonic and instCode
        if instCode == None:
            kinstList = fileInfo.getKinstList()
            if len(kinstList) == 0:
                raise ValueError('No kinst values found in file')
            if len(kinstList) > 1:
                raise ValueError('More than one kinst value found in file: %s' % (str(kinstList)))
            instCode = kinstList[0]
        instMnemonic = self.__madInst.getInstrumentMnemonic(instCode)
        instDesc = self.__madInst.getInstrumentName(instCode)
        if instMnemonic == None:
            raise ValueError('Unable to find mnemonic for kinst %i' % (instCode))

        # expTitle
        if type(expTitle) not in (bytes, str) and expTitle != None:
            raise ValueError('expTitle not a string')
        if expTitle == None:
            expTitle = ''
        if expTitle.find(',') != -1:
            raise ValueError('expTitle cannot contain a comma')

        # kindat
        if kindat == None:
            kindatList = fileInfo.getKindatList()
            if len(kindatList) == 0:
                raise ValueError('No kindat values found in file')
            if len(kindatList) > 1:
                raise ValueError('More than one kindat value found in file: %s' % (str(kindatList)))
            kindat = kindatList[0]
        else:
            kindat = int(kindat)

        # permission
        if permission not in (0,1, -1):
            raise ValueError('permission must be either 0 or 1 or -1, not %s' % (str(permission)))

        # fileDesc
        if type(fileDesc) not in  (bytes, str) and fileDesc != None:
            raise ValueError('fileDesc not a string')
        if fileDesc == None:
            fileDesc = ''
        if fileDesc.find(',') != -1:
            raise ValueError('fileDesc cannot contain a comma')

        # category
        if category not in (1,2,3,4):
            raise ValueError('category must be 1=default, 2=variant, 3=history, or 4=realtime; not %s' % (str(category)))

        # hasCatalog and hasHeader
        catStr = fileInfo.getCatalogHeaderStr()
        if len(catStr) > 0:
            hasCatalog = 1
            hasHeader = 1
        else:
            hasCatalog = 0
            hasHeader = 0
            
        # experimentDirNum
        if experimentsDirNum != None:
            experimentsDir = 'experiments%i' % (experimentsDirNum)
            # verify this directory exists
            if not os.access(os.path.join(self.__madDB.getMadroot(), experimentsDir), os.R_OK):
                raise ValueError('no such directory %s' % (experimentsDir))
        else:
            experimentsDir = 'experiments'
            
        # PI
        if type(PI) not in (bytes, str) and PI != None:
            raise ValueError('PI not a string')
        if PI == None:
            PI = ''
        if PI.find(',') != -1:
            raise ValueError('PI cannot contain a comma')
            
        # PIEmail
        if type(PIEmail) not in (bytes, str) and PIEmail != None:
            raise ValueError('PIEmail not a string')
        if PIEmail == None:
            PIEmail = ''
        if PIEmail.find(',') != -1:
            raise ValueError('PIEmail cannot contain a comma')
            
        # fileAnalyst
        if type(fileAnalyst) not in (bytes, str) and fileAnalyst != None:
            raise ValueError('fileAnalyst not a string')
        if fileAnalyst == None:
            fileAnalyst = ''
        if fileAnalyst.find(',') != -1:
            raise ValueError('fileAnalyst cannot contain a comma')
            
        # fileAnalystEmail
        if type(fileAnalystEmail) not in (bytes, str) and fileAnalystEmail != None:
            raise ValueError('fileAnalystEmail not a string')
        if fileAnalystEmail == None:
            fileAnalystEmail = ''
        if fileAnalystEmail.find(',') != -1:
            raise ValueError('fileAnalystEmail cannot contain a comma')
        
        # all the arguments check out - create the directory
        # create experiment dir
        expDir2 = os.path.join('%i' % (startTime.year),
                              instMnemonic)
        
        if dirName == None:
            dirName = startTime.strftime('%d%b%y').lower() + optChar
        else:
            # verify dirName is basename
            if dirName.find('/') != -1:
                raise ValueError('dirName must be base directory name, not %s' % (dirName))
            
        expDir2 = os.path.join(expDir2, dirName)

        expDir = os.path.join(self.__madDB.getMadroot(), experimentsDir, expDir2)
        
        expDir3 = os.path.join(experimentsDir, expDir2)

        # if the directory already exists, raise error
        if os.access(expDir, os.R_OK):
            raise IOError('Directory %s already exists' % (expDir))

        os.makedirs(expDir)
        os.chmod(expDir, 0o775)

        # expTab.txt
        expTabText = '0,' + self.__madDB.getTopLevelUrl()
        if expTabText[-1] != '/':
            expTabText += '/madtoc/'
        else:
            expTabText += 'madtoc/'
        expTabText += '%s,%s,%i,%04i%02i%02i,%02i%02i%02i,%04i%02i%02i,%02i%02i%02i,%i,%i,%s,%s\n' %(expDir3,
                                                                                                     expTitle,
                                                                                                     self.__madDB.getSiteID(),
                                                                                                     startTime.year,
                                                                                                     startTime.month,
                                                                                                     startTime.day,
                                                                                                     startTime.hour,
                                                                                                     startTime.minute,
                                                                                                     startTime.second,
                                                                                                     endTime.year,
                                                                                                     endTime.month,
                                                                                                     endTime.day,
                                                                                                     endTime.hour,
                                                                                                     endTime.minute,
                                                                                                     endTime.second,
                                                                                                     instCode,
                                                                                                     permission,
                                                                                                     PI, PIEmail)

        # write expTab.txt
        f = open(os.path.join(expDir, 'expTab.txt'), 'w', encoding='utf-8')
        f.write(expTabText)
        f.close()
        os.chmod(os.path.join(expDir, 'expTab.txt'), 0o664)

        # fileTab.txt
         # set file times to right now UT
        nowUT = datetime.datetime.utcnow()
        nowDate = nowUT.strftime('%Y%m%d')
        nowTime = nowUT.strftime('%H%M%S')
        fileTabText = os.path.basename(madFilename)
        fileTabText += ',0,%i,%i,0,%i,%i,%s,%s,' % (kindat,category,hasCatalog,hasHeader,nowDate,nowTime)
        fileTabText += '%s,%i,%s,%s\n' % (fileDesc, permission,
                                          fileAnalyst, fileAnalystEmail)

        # write fileTab.txt
        f = open(os.path.join(expDir, 'fileTab.txt'), 'w', encoding='utf-8')
        f.write(fileTabText)
        f.close()
        os.chmod(os.path.join(expDir, 'fileTab.txt'), 0o664)

        # create all writeable directory overview
        os.makedirs(os.path.join(expDir, 'overview'))
        os.chmod(os.path.join(expDir, 'overview'), 0o777)

        # cp madFilename to new directory
        shutil.copy2(madFilename, os.path.join(expDir, os.path.basename(madFilename)))
        os.chmod(os.path.join(expDir, os.path.basename(madFilename)), 0o664)

        # populate overview
        overviewFile = os.path.join(os.path.dirname(madFilename), 'overview', os.path.basename(madFilename) + '.summary')
        if os.access(overviewFile, os.R_OK):
            # make sure overview exist
            overviewDir = os.path.join(expDir, 'overview')
            if not os.access(overviewDir, os.W_OK):
                os.mkdir(overviewDir)
                os.chmod(overviewDir, 0o777)
            shutil.copy2(overviewFile, overviewDir)
        else:
            fileInfo = madrigal.data.MadrigalFile(os.path.join(expDir, os.path.basename(madFilename)), self.__madDB,
                                                  acceptOldSummary=acceptOldSummary)
        
        if createCachedText:
            cachedName = os.path.join(expDir, 'overview', os.path.basename(madFilename) + '.txt')
            madrigal.cedar.convertToText(madFilename, cachedName)
            subprocess.check_call(['gzip', cachedName])
            
        if createCachedNetCDF4:
            cachedName = os.path.join(expDir, 'overview', os.path.basename(madFilename) + '.nc')
            madrigal.cedar.convertToNetCDF4(madFilename, cachedName)
        
        if notify:
            # get expPath without MAD ROOT
            expPath = expDir[expDir.find('experiments'):]
            if expPath[-1] == '/':
                expPath = expPath[:-1]
            madNotify = MadrigalNotify(self.__madDB)
            message = 'You requested to be notified when the instrument %s from the Madrigal site %s was updated. A new experiment at %s was created on %s.'  \
                % (instDesc, self.__madDB.getTopLevelUrl(), expDir3, str(datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S')))
            message += 'Please contact %s if you wish to be unregistered.' % (self.__madDB.getContactEmail())
                
            # deal with users registered for instrument in general
            userList = self._userData.getRegisteredInstUsers(instCode)
            if len(userList) > 0:
                for user in userList:
                    try:
                        madNotify.notify(user, message, 'Update to Madrigal instrument data you registered interest in')
                    except:
                        print(('Warning - email to %s failed to be sent' % (user)))

        return(expDir)



    def changeExpStatus(self,
                        expDir,
                        expUrl=None,
                        expName = None,
                        siteID = None,
                        startDatetime = None,
                        endDatetime = None,
                        inst = None,
                        security = None,
                        PI = None,
                        PIEmail = None):
        """changeExpStatus is used to change attributes in expTab.txt.  If None, no change.

        Inputs:
        
            expDir - full path to experiment directory. Required.  Example:
               "/opt/madrigal/experiments/1998/mlh/20jan98". If None, do not change.
            
            expUrl - must be in form /madtoc/YYYY/<3 letter lower case inst code>/
                       example: http://www.haystack.mit.edu/cgi-bin/madtoc/1997/mlh/03dec97g.
                       If None, do not change.

            expName - experiment name.  Quotes required if contains spaces.  Example: "World Day"
                        If None, do not change.

            siteID - Madrigal siteID (int) of where data will be stored.  Error raised if not the siteID
                       of the local Madrigal site. Example: 4. If None, do not change.

            startDatetime - new start datetime of experiment (UT). If None, do not change.

            endDatetime - new end datetime of experiment (UT). If None, do not change.

            inst - new instrument code (int).  Example: 30. If None, do not change.  Prints
                    warning if not found in instTab.txt

            security - new security code.  Allowed values are 0 for public, 1 for private (limited IP range access)
                    -1 for ignore, 2 for archived experiment, 3 for private (limited IP range access) archived
                    experiment. If None, do not change.
                    
            PI - name of PI. If None, no change
            
            PIEmail - PI email.  If None, no change
            
        """
        try:
            expTabInfo = madrigal.metadata.MadrigalExperiment(self.__madDB, os.path.join(expDir, 'expTab.txt'))
        except:
            raise ValueError('Unable to open expTab.txt in %s' % (expDir))

        # be sure only one experiment
        if expTabInfo.getExpCount() != 1:
            raise ValueError('expTab.txt in %s has %i experiments, should have exactly 1' % (expDir,
                                                                                              expTabInfo.getExpCount()))

        # expUrl
        if expUrl != None:
            # print warning if not this directory
            index = expUrl.find('/madtoc/')
            thisDir = expUrl[index+8:]
            if thisDir[-1] == '/':
                thisDir = thisDir[:-1]
            if expDir.find(thisDir) == -1:
                raise ValueError('The experiment url you are setting this experiment to <%s> conflicts with experiment directory %s' % (expUrl, expDir))
            expTabInfo.setExpUrlByPosition(0, expUrl)

        # expName
        if expName != None:
            expTabInfo.setExpNameByPosition(0, expName)

        # siteID
        if siteID != None:
            siteID = int(siteID)
            if siteID != self.__madDB.getSiteID():
                raise ValueError('Setting experiment to a siteID %i different from this site\'s id %i' % (siteID,
                                                                                                                    self.__madDB.getSiteID()))
            expTabInfo.setExpSiteIdByPosition(0, siteID)

        # startDatetime
        if startDatetime != None:
            # verify before endDatetime if that also being set
            if endDatetime != None:
                if startDatetime > endDatetime:
                    raise ValueError('startDatetime %s must be before endDatetime %s' % (str(startDatetime),
                                                                                          str(endDatetime)))
            expTabInfo.setExpStartDateTimeByPosition(startDatetime, 0)

        # endDatetime
        if endDatetime != None:
            expTabInfo.setExpEndDateTimeByPosition(endDatetime, 0)

        # inst
        if inst != None:
            inst = int(inst)
            if self.__madInst.getInstrumentName(inst) == None:
                print('WARNING: instrument %i not found in instTab.txt' % (inst))
            expTabInfo.setExpKinstByPosition(0, inst)

        # security
        if security != None:
            security = int(security)
            if security not in (-1, 0, 1, 2, 3):
                raise ValueError('security must be in (-1, 0, 1, 2, 3), not %i' % (inst))
            expTabInfo.setSecurityByPosition(0, security)
            
        # PI
        if PI != None:
            expTabInfo.setPIByPosition(0, PI)
            
        # PIEmail
        if PIEmail != None:
            expTabInfo.setPIEmailByPosition(0, PIEmail)

        # everything successfully changed - write new values
        expTabInfo.writeMetadata()
        
            

    def addMadrigalFile(self,
                        expDir,
                        madFilename,
                        permission,
                        fileDesc,
                        category = 1,
                        kindat = None,
                        notify = True,
                        fileAnalyst = '',
                        fileAnalystEmail = '',
                        createCachedText=False, createCachedNetCDF4=False,
                        updateToMad3=False, acceptOldSummary=False):
        """addMadrigalFile adds a new file to an experiment using metadata read from madFilename.

        Inputs:

            expDir - full path to experiment directory (as returned by createMadriogalExperiment)
        
            madFilename - full path to the complete Madrigal file.  Basename will be maintained.
            
            permission - 0 (public) or 1 (private). 
            
            fileDesc - file description

            category - 1=default, 2=variant, 3=history, or 4=realtime. Default is 1 (default file)

            kindat - if not None (the default), use this kindat instead of what is found in the file.
            
            notify - if True (the default), send a message to all registered users.  If False, do not.
            
            fileAnalyst - full name of file Analyst.  Default is ''
            
            fileAnalystEmail - email of file Analyst.  Default is ''
            
            createCachedText - if True, add cached text file in overview/.txt.gz.  If False,
                no cached file.
                
            createCachedNetCDF4 - if True, add cached netCDF4 file in overview/.nc.  If False,
                no cached file.
              
            updateToMad3 - if False (the default), error raised if madFilename non-Hdf5 file. If True, try to
                convert madFilename to Madrigal with .hdf5 extension before loading.
                
            acceptOldSummary - if True, accept an old summary file. Used mainly for upgrading to Madrigal 3. Default
                is False.

        Returns: None
            
        """
        base, ext = os.path.splitext(madFilename)
        if updateToMad3:
            if ext not in ('.hdf5', '.h5', '.hdf5'):
                madFilename = convertMad2FileToMad3(madFilename, self.__madDB, fileDesc)
        elif ext not in ('.hdf5', '.h5', '.hdf5'):
            raise ValueError('called addMadrigalFile with non-Hdf5 file <%s> and updateToMad3 False' \
                % (madFilename))
                
        fileInfo = madrigal.data.MadrigalFile(madFilename, self.__madDB, acceptOldSummary=acceptOldSummary)

        # kindat
        if kindat == None:
            kindatList = fileInfo.getKindatList()
            if len(kindatList) == 0:
                raise ValueError('No kindat values found in file')
            if len(kindatList) > 1:
                raise ValueError('More than one kindat value found in file: %s' % (str(kindatList)))
            kindat = kindatList[0]
        else:
            kindat = int(kindat)

        # permission
        if permission not in (0,1):
            raise ValueError('permission must be either 0 or 1, not %s' % (str(permission)))

        # fileDesc
        if type(fileDesc) not in (bytes, str) and fileDesc != None:
            raise ValueError('fileDesc not a string')
        if fileDesc == None:
            fileDesc = ''
        # check that fileDesc does not illegally contain a comma
        if fileDesc.find(',') != -1:
            raise ValueError('fileDesc string in fileTab.txt cannot contain a comma: <%s> is illegal' % (fileDesc))

        # category
        if category not in (1,2,3,4):
            raise ValueError('category must be 1=default, 2=variant, 3=history, or 4=realtime; not %s' % (str(category)))

        # hasCatalog and hasHeader
        catStr = fileInfo.getCatalogHeaderStr()
        if len(catStr) > 0:
            hasCatalog = 1
            hasHeader = 0
        else:
            hasCatalog = 0
            hasHeader = 0
            
        if fileAnalyst.find(',') != -1:
            raise ValueError('fileAnalyst cannot contain a comma')
        
        if fileAnalystEmail.find(',') != -1:
            raise ValueError('fileAnalystEmail cannot contain a comma')
        

        # all the arguments check out - add line to fileTab.txt
        if not os.access(os.path.join(expDir, 'fileTab.txt'), os.R_OK):
            raise ValueError(' file %s does not yet exist' % (os.path.join(expDir, 'fileTab.txt')))

        # check that this is a new filename
        fileTabObj = madrigal.metadata.MadrigalMetaFile(self.__madDB, os.path.join(expDir, 'fileTab.txt'))
        for i in range(fileTabObj.getFileCount()):
            filename = fileTabObj.getFilenameByPosition(i)
            if filename == os.path.basename(madFilename):
                raise ValueError('File %s already exists - must be deleted first' % (filename))
            

        # fileTab.txt
        nowUT = datetime.datetime.utcnow()
        nowDate = nowUT.strftime('%Y%m%d')
        nowTime = nowUT.strftime('%H%M%S')
        fileTabText = os.path.basename(madFilename)
        fileTabText += ',0,%i,%i,0,%i,%i,%s,%s,' % (kindat,category,hasCatalog,hasHeader,nowDate,nowTime)
        fileTabText += '%s,%i,%s,%s\n' % (fileDesc, permission,
                                          fileAnalyst, fileAnalystEmail)

        # write fileTab.txt
        f = open(os.path.join(expDir, 'fileTab.txt'), 'a')
        f.write(fileTabText)
        f.close()                                                                                                                

        # cp madFilename to new directory
        shutil.copy2(madFilename, os.path.join(expDir, os.path.basename(madFilename)))
        os.chmod(os.path.join(expDir, os.path.basename(madFilename)), 0o664)

        # populate overview
        overviewFile = os.path.join(os.path.dirname(madFilename), 'overview', os.path.basename(madFilename) + '.summary')
        if os.access(overviewFile, os.R_OK):
            # make sure overview exist
            overviewDir = os.path.join(expDir, 'overview')
            if not os.access(overviewDir, os.W_OK):
                os.mkdir(overviewDir)
                os.chmod(overviewDir, 0o777)
            shutil.copy2(overviewFile, overviewDir)
        else:
            fileInfo = madrigal.data.MadrigalFile(os.path.join(expDir, os.path.basename(madFilename)), self.__madDB,
                                                  acceptOldSummary=acceptOldSummary)
                
        if createCachedText:
            cachedName = os.path.join(expDir, 'overview', os.path.basename(madFilename) + '.txt')
            madrigal.cedar.convertToText(madFilename, cachedName)
            subprocess.check_call(['gzip', cachedName])
            
        if createCachedNetCDF4:
            cachedName = os.path.join(expDir, 'overview', os.path.basename(madFilename) + '.nc')
            madrigal.cedar.convertToNetCDF4(madFilename, cachedName)

        # update expTab.txt against all registered files
        self.updateExpTab(expDir)
        
        
        if notify:
            # get expPath without MAD ROOT
            expPath = expDir[expDir.find('experiments'):]
            if expPath[-1] == '/':
                expPath = expPath[:-1]
            madNotify = MadrigalNotify(self.__madDB)
            message = 'You requested to be notified when the experiment: %s from the Madrigal site %s was updated. A new file %s has been added to that experiment at %s. '  \
                % (expPath, self.__madDB.getTopLevelUrl(), os.path.basename(madFilename), str(datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S')))
            message += 'Please contact %s if you wish to be unregistered.' % (self.__madDB.getContactEmail())
                
            # first deal with users registered for just this experiment
            userList = self._userData.getRegisteredUsers(expPath)
            if len(userList) > 0:
                for user in userList:
                    try:
                        madNotify.notify(user, message, 'Update to Madrigal experiment you registered interest in')
                    except:
                        print(('Warning - email to %s failed to be sent' % (user)))
                        
            # next deal with users registered for instrument in general
            madExpObj = madrigal.metadata.MadrigalExperiment(self.__madDB, os.path.join(expDir, 'expTab.txt'))
            kinst = madExpObj.getKinstByPosition(0)
            userList = self._userData.getRegisteredInstUsers(kinst)
            if len(userList) > 0:
                for user in userList:
                    try:
                        madNotify.notify(user, message, 'Update to Madrigal instrument data you registered interest in')
                    except:
                        print(('Warning - email to %s failed to be sent' % (user)))
            



    def changeFileStatus(self,
                         expDir,
                         filename,
                         category = None,
                         fileDesc = None,
                         permission = None,
                         kindat = None,
                         fileAnalyst = None,
                         fileAnalystEmail = None):
        """changeFileStatus is used to change category, fileDesc, or permission of a register file in fileTab.txt.

        Inputs:

            expDir - full path to experiment directory 
        
            filename - basename of existing Madrigal file already registered in fileTab.txt
            
            permission - 0 (public) or 1 (private). If None (default), leave unchanged. 
            
            fileDesc - file description. If None (default), leave unchanged.

            category - 1=default, 2=variant, or 3=history. If None (default), leave unchanged.

            kindat - kindat (int). If None (default), leave unchanged.
            
            fileAnalyst - name of file analyst.  If None (default), leave unchanged.
            
            fileAnalystEmail - email of file analyst.  If None (default), leave unchanged.
        """
        try:
            fileTabInfo = madrigal.metadata.MadrigalMetaFile(self.__madDB, os.path.join(expDir, 'fileTab.txt'))
        except:
            raise ValueError('Unable to open fileTab.txt in %s' % (expDir))

        # be sure filename is basename
        filename = os.path.basename(filename)

        # search all Madrigal files in fileTab.txt for right file
        found = False
        fileCount = fileTabInfo.getFileCount()

        for index in range(fileCount):
            thisFilename = fileTabInfo.getFilenameByPosition(index)
            if thisFilename == filename:
                found = True
                if category != None:
                    fileTabInfo.setCategoryByPosition(index, category)
                if fileDesc != None:
                    fileTabInfo.setStatusByPosition(index, fileDesc)
                if permission != None:
                    fileTabInfo.setAccessByPosition(index, permission)
                if kindat != None:
                    fileTabInfo.setKindatByPosition(index, kindat)
                if fileAnalyst != None:
                    fileTabInfo.setAnalystByPosition(index, fileAnalyst)
                if fileAnalystEmail != None:
                    fileTabInfo.setAnalystEmailByPosition(index, fileAnalystEmail)
                fileTabInfo.writeMetadata()
                break

        if found == False:
            raise ValueError('Madrigal file %s not found in %s' % (filename, os.path.join(expDir, 'fileTab.txt')))


    def overwriteMadrigalFile(self,
                              expDir,
                              madFilename,
                              notify = True):
        """overwriteMadrigalFile overwrites a file already registered in fileTab.txt.

        Automatically updates expTab.txt with any start or end experiment times.

        Inputs:

            expDir - full path to experiment directory
        
            madFilename - full path to the new Madrigal file.  Basename must match that of one in fileTab.txt.
            
            notify - if True (the default), send a message to all registered users.  If False, do not.
    

        Returns: None

        Affects: Overwrites existing Madrigal file.  May modify expTab.txt with new start/end times.
        Also updated any cached files.
        """
        # verify this file registered
        try:
            fileTabInfo = madrigal.metadata.MadrigalMetaFile(self.__madDB, os.path.join(expDir, 'fileTab.txt'))
        except:
            raise ValueError('Unable to open fileTab.txt in %s' % (expDir))

        # get basename
        filename = os.path.basename(madFilename)

        # search all Madrigal files in fileTab.txt for right file
        found = False
        fileCount = fileTabInfo.getFileCount()

        for index in range(fileCount):
            thisFilename = fileTabInfo.getFilenameByPosition(index)
            if thisFilename == filename:
                found = True
                break

        if found == False:
            raise ValueError('%s not found in %s' % (filename, os.path.join(expDir, 'fileTab.txt')))

        # cp madFilename to new directory
        shutil.copy(madFilename, os.path.join(expDir, filename))
        os.chmod(os.path.join(expDir, filename), 0o664)

        # rm and re-populate overview
        os.remove(os.path.join(expDir, 'overview', filename + '.summary'))
        fileInfo = madrigal.data.MadrigalFile(os.path.join(expDir, filename), self.__madDB)
        
        cachedName = os.path.join(expDir, 'overview', filename + '.txt')
        if os.access(cachedName + '.gz', os.R_OK):
            os.remove(cachedName + '.gz')
            madrigal.cedar.convertToText(os.path.join(expDir, filename), cachedName)
            subprocess.check_call(['gzip', cachedName])
            
        cachedName = os.path.join(expDir, 'overview', filename + '.nc')
        if os.access(cachedName, os.R_OK):
            os.remove(cachedName)
            madrigal.cedar.convertToNetCDF4(os.path.join(expDir, filename), cachedName)

        # update expTab.txt against all registered files
        self.updateExpTab(expDir)
        
        if notify:
            # get expPath without MAD ROOT
            expPath = expDir[expDir.find('experiments'):]
            if expPath[-1] == '/':
                expPath = expPath[:-1]
            madNotify = MadrigalNotify(self.__madDB)
            message = 'You requested to be notified when the experiment: %s from the Madrigal site %s was updated. The file %s has been modified in that experiment at %s.'  \
                % (expPath, self.__madDB.getTopLevelUrl(), os.path.basename(madFilename), str(datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S')))
            message += 'Please contact %s if you wish to be unregistered.' % (self.__madDB.getContactEmail())
            
            userList = self._userData.getRegisteredUsers(expPath)
            if len(userList) > 0:
                for user in userList:
                    try:
                        madNotify.notify(user, message, 'Update to Madrigal experiment you registered interest in')
                    except:
                        print(('Warning - email to %s failed to be sent' % (user)))
                        
            # next deal with users registered for instrument in general
            madExpObj = madrigal.metadata.MadrigalExperiment(self.__madDB, os.path.join(expDir, 'expTab.txt'))
            kinst = madExpObj.getKinstByPosition(0)
            userList = self._userData.getRegisteredInstUsers(kinst)
            if len(userList) > 0:
                for user in userList:
                    try:
                        madNotify.notify(user, message, 'Update to Madrigal instrument data you registered interest in')
                    except:
                        print(('Warning - email to %s failed to be sent' % (user)))



    def removeMadrigalFile(self,
                           expDir,
                           madFilename,
                           allowMissing=False):
        """removeMadrigalFile removes a file already registered in fileTab.txt.

        Automatically updates expTab.txt with any start or end experiment times.

        Inputs:

            expDir - full path to experiment directory
        
            madFilename - Name of Madrigal file to be removed.  Basename must match that of one in fileTab.txt.
            
            allowMissing - if True, remove file successfully even if the file cannot be removed from expDir. If False,
                the default, raise an error if file does not exist.

        Returns: None

        Affects: Removes existing Madrigal file and removes its line from fileTab.txt.  May modify expTab.txt
        with new start/end times
        """
        # verify this file registered
        try:
            fileTabInfo = madrigal.metadata.MadrigalMetaFile(self.__madDB, os.path.join(expDir, 'fileTab.txt'))
        except:
            raise ValueError('Unable to open fileTab.txt in %s' % (expDir))

        # get basename
        filename = os.path.basename(madFilename)

        # delete line from fileTab.txt
        fileTabInfo.deleteRowByFilename(filename)

        # write new version
        fileTabInfo.writeMetadata()
        
        # state variable to detect deletion of missing file
        was_missing = False

        # rm filename and overview data
        try:
            os.remove(os.path.join(expDir, filename))
        except:
            if allowMissing:
                print(('Warning: Unable to remove %s' % (os.path.join(expDir, filename))))
                was_missing = True
            else:
                raise
        try:
            os.remove(os.path.join(expDir, 'overview', filename + '.summary'))
        except:
            pass
        
        # sometimes conversion to Madrigal 3 left a copy in overview - remove this too if its there
        try:
            os.remove(os.path.join(expDir, 'overview', filename))
        except:
            pass
        
        
        cachedName = os.path.join(expDir, 'overview', filename + '.txt')
        if os.access(cachedName + '.gz', os.R_OK):
            os.remove(cachedName + '.gz')
            
        cachedName = os.path.join(expDir, 'overview', filename + '.nc')
        if os.access(cachedName, os.R_OK):
            os.remove(cachedName)

        # update expTab.txt against all registered files
        if not was_missing:
            self.updateExpTab(expDir)
        
        



    def addWebFile(self,
                   expDir,
                   source,
                   relativePath):
        """addWebFile writes a non-Madrigal file meant to be displayed on the web to somewhere within a Madrigal experiment directory.

            All needed directories will be created if needed.

            Inputs:

                expDir - full path to experiment directory

                source - local web file to write to Madrigal

                relativePath - path relative to expDir to write source file to.  If relativePath ends
                with /, then basename from source used.  Otherwise, basename from relativePath used.

            Returns: None

            Affects: writes a non-Madrigal file to expDir on Madrigal
        """
        # verify expDir is a real Madrigal experiment directory
        try:
            fileTabInfo = madrigal.metadata.MadrigalMetaFile(self.__madDB, os.path.join(expDir, 'fileTab.txt'))
        except:
            raise ValueError('%s not a valid experiment directory - no fileTab.txt' % (expDir))

        if not os.access(os.path.join(expDir, os.path.dirname(relativePath)), os.R_OK):
            # make all dirs
            os.umask(0000)
            os.makedirs(os.path.join(expDir, os.path.dirname(relativePath)), 0o777)
            
        if os.path.basename(relativePath) == '':
            shutil.copy(source, os.path.join(expDir, relativePath, os.path.basename(source)))
            os.chmod(os.path.join(expDir, os.path.dirname(relativePath), os.path.basename(source)), 0o664)
        else:
            shutil.copy(source, os.path.join(expDir, relativePath))
            os.chmod(os.path.join(expDir, relativePath), 0o664)

        

    def updateExpTab(self, expDir):
        """updateExpTab rewrites expTab.txt based on all the Madrigal files registered in fileTab.txt.

        Inputs:

            expDir - full path to experiment directory

        Returns: None

        Affects: rewrites expTab.txt in expDir based on all the Madrigal files registered in fileTab.txt.
        """
        try:
            fileTabInfo = madrigal.metadata.MadrigalMetaFile(self.__madDB, os.path.join(expDir, 'fileTab.txt'))
        except:
            raise ValueError('Unable to open fileTab.txt in %s' % (expDir))

        try:
            expInfo = madrigal.metadata.MadrigalExperiment(self.__madDB, os.path.join(expDir, 'expTab.txt'))
        except:
            raise ValueError('Unable to open expTab.txt in %s' % (expDir))

        # we only need to modify start and end times of experiment
        startTime = None
        endTime = None
        # we'd perfer to use default times if possible
        startTimeDefault = None
        endTimeDefault = None

        # search all Madrigal files for earliest time, latest time
        fileCount = fileTabInfo.getFileCount()
        if fileCount == 0:
            return

        for index in range(fileCount):
            thisMadfilename = fileTabInfo.getFilenameByPosition(index)
            thisCategory = fileTabInfo.getCategoryByPosition(index)
            try:
                fileInfo = madrigal.data.MadrigalFile(os.path.join(expDir, thisMadfilename))
            except:
                print('WARNING: Problem with expDir=%s and thisMadfilename=%s' % (expDir, thisMadfilename))
                continue
            thisStartTimeList = fileInfo.getEarliestTime()
            thisStartTime = datetime.datetime(thisStartTimeList[0],
                                              thisStartTimeList[1],
                                              thisStartTimeList[2],
                                              thisStartTimeList[3],
                                              thisStartTimeList[4],
                                              thisStartTimeList[5])
            thisEndTimeList = fileInfo.getLatestTime()
            thisEndTime = datetime.datetime(thisEndTimeList[0],
                                            thisEndTimeList[1],
                                            thisEndTimeList[2],
                                            thisEndTimeList[3],
                                            thisEndTimeList[4],
                                            thisEndTimeList[5])

            if startTime == None:
                startTime = thisStartTime
            else:
                if startTime > thisStartTime:
                    startTime = thisStartTime

            if endTime == None:
                endTime = thisEndTime
            else:
                if endTime < thisEndTime:
                    endTime = thisEndTime
                    
            if thisCategory == 1:
                if startTimeDefault == None:
                    startTimeDefault = thisStartTime
                else:
                    if startTimeDefault > thisStartTime:
                        startTimeDefault = thisStartTime
    
                if endTimeDefault == None:
                    endTimeDefault = thisEndTime
                else:
                    if endTimeDefault < thisEndTime:
                        endTimeDefault = thisEndTime

        # modify expTab.txt and write
        if startTimeDefault:
            expInfo.setExpStartDateTimeByPosition(startTimeDefault)
        else:
            expInfo.setExpStartDateTimeByPosition(startTime)
        if endTimeDefault:
            expInfo.setExpEndDateTimeByPosition(endTimeDefault)
        else:
            expInfo.setExpEndDateTimeByPosition(endTime)
        expInfo.writeMetadata()


    def updateMaster(self, skipGeo=False):
        """updateMaster is a method to update the local metadata.

        Replaces the former tcl script.

        Gathers data from experiment directories into metadata/expTab.txt and metadata/fileTab.txt.
        Also gathers metadata from OpenMadrigal to update metadata/expTabAll.txt and metadata/fileAllTab.txt,
        to update high level metadata siteTab.txt, instTab.txt, instType.txt, madCatTab.txt, parcods.tab.
        Also updates geophysical data.
        """
        binDir = os.path.join(self.__madDB.getMadroot(), 'bin')

        # update geophysical data
        if not skipGeo:
            print('*** Checking for any geophysical file updates ***')
            cmd = os.path.join(binDir, 'checkGeoUpdate.py')
            os.system(cmd)
        else:
            print('Warning - skipping updating geophysical files too often will make them out of date.')
        
        print('*** Updating local metadata ***')
        self.__updateLocalMetadata__()
        print('*** Updating metadata from other Madrigal sites ***')
        self.__updateGlobalMetadata__()
        print('*** Checking OpenMadrigal for any metadata updates ***')
        self.__checkOpenMadrigalMetadata__()

        # instParmTab.txt
        print('*** Rebuilding instParmTab.txt ***')
        obj = madrigal.metadata.MadrigalInstrumentParameters(self.__madDB)
        obj.rebuildInstParmTable()
        
        # instKindatTab.txt
        print('*** Rebuilding instKindatTab.txt ***')
        obj = madrigal.metadata.MadrigalInstrumentKindats(self.__madDB)
        obj.rebuildInstKindatTable()

        print('updateMaster complete...')



    def __updateLocalMetadata__(self):
        """__updateLocalMetadata__ is a private method to update metadata/expTab.txt and metadata/fileTab.txt
        from the local metadata in the experiments[0-9]* directory
        """
        localSiteID = self.__madDB.getSiteID() # used to check that experiments do not have wrong siteID
        
        metaDict = {}
        metaDict['expText'] = [] # a list of text lines of combined expTab.txt file
        metaDict['expIds'] = [] # a list of all expIds found in locals dirs
        metaDict['expIds2'] = [] # a list of all expIds found summary expDir.txt
        metaDict['fileText'] = [] # a list of text lines of combined fileTab.txt file
        metaDict['presentCount'] = 0 # experiment count so far
        metaDict['totalCount'] = 0 # count of all experiments found in first count
        metaDict['localSiteId'] = localSiteID
        
        # make sure metaDict['expIds'] and metaDict['expIds2'] contains minimum value
        metaDict['expIds'].append(self.__madDB.getSiteID() * 10000000)
        metaDict['expIds2'].append(self.__madDB.getSiteID() * 10000000)
        
        # get all experiment directories
        expDirList = self.__madDB.getExperimentDirs()
        
        # first walk to to simply fill out metaDict['expIds']
        for thisExpDir in expDirList:
            if not os.path.isdir(thisExpDir):
                continue
            for root, dirs, files, in os.walk(thisExpDir):
                         self.__walkExpDirIds__(metaDict, root, dirs + files)
        
        for thisExpDir in expDirList:
            if not os.path.isdir(thisExpDir):
                continue
            for root, dirs, files in os.walk(thisExpDir):
                         self.__walkExpDir__(metaDict, root, dirs + files)

        # update expTab.txt
        f = open(os.path.join(self.__madDB.getMadroot(), 'metadata/expTab.txt'), 'w', encoding='utf-8')
        delimiter = ''
        f.write(delimiter.join(metaDict['expText']))
        f.close()
        
        # now sort it by date to speed searches
        madExpObj = madrigal.metadata.MadrigalExperiment(self.__madDB)
        madExpObj.sortByDateSite()
        madExpObj.writeMetadata(os.path.join(self.__madDB.getMadroot(), 'metadata/expTab.txt'))

        # update fileTab.txt
        f = open(os.path.join(self.__madDB.getMadroot(), 'metadata/fileTab.txt'), 'w', encoding='utf-8')
        f.write(delimiter.join(metaDict['fileText']))
        f.close()


    def __updateGlobalMetadata__(self):
        """__updateGlobalMetadata__ is a private method to update metadata/expTabAll.txt and metadata/fileTabAll.txt
        from the main madrigal server.
        """

        expTabAll = ''
        fileTabAll = ''

        localSiteID = self.__madDB.getSiteID()

        siteList = self.__madSite.getSiteList()

        for site in siteList:
            siteID = site[0]
            # skip local site
            if siteID == localSiteID:
                continue
            siteName = site[1]
            siteDir = '%s_%i' % (siteName, siteID)

            expMetadataFile = os.path.join(siteDir, 'expTab.txt')
            fileMetadataFile = os.path.join(siteDir, 'fileTab.txt')

            try:
                thisExpText  = self.__openMad.getMetadataFromOpenMadrigal(expMetadataFile)
                thisFileText = self.__openMad.getMetadataFromOpenMadrigal(fileMetadataFile)
            except:
                continue

            expTabAll += thisExpText
            fileTabAll += thisFileText

        # append local data
        f = open(os.path.join(self.__madDB.getMadroot(), 'metadata/expTab.txt'))
        expTabAll += f.read()
        f.close()

        f = open(os.path.join(self.__madDB.getMadroot(), 'metadata/fileTab.txt'))
        fileTabAll += f.read()
        f.close()

        # write *All.txt files
        f = open(os.path.join(self.__madDB.getMadroot(), 'metadata/expTabAll.txt'), 'w', encoding='utf-8')
        f.write(expTabAll)
        f.close()
        
        # now sort it by date to speed searches
        madExpObj = madrigal.metadata.MadrigalExperiment(self.__madDB,
                                                         os.path.join(self.__madDB.getMadroot(), 
                                                                      'metadata/expTabAll.txt'))
        madExpObj.sortByDateSite()
        madExpObj.writeMetadata(os.path.join(self.__madDB.getMadroot(), 'metadata/expTabAll.txt'))


        f = open(os.path.join(self.__madDB.getMadroot(), 'metadata/fileTabAll.txt'), 'w', encoding='utf-8')
        f.write(fileTabAll)
        f.close()
        
        # rewrite instData.txt
        self._updateInstData()


    def __checkOpenMadrigalMetadata__(self):
        """__checkOpenMadrigalMetadata__ is a method that check the openmadrigal site for any
        updates to the following metadata files:

            1. siteTab.txt - the list of all Madrigal installations
            2. instTab.txt - the list of all Madrigal instruments
            3. instType.txt - the list of all instrument categories

        If an update is available, and the existing metadata file is an old one, it will be updated.
        However, if the local Madrigal adminstrator edits one of these files, then the file will
        not be updated.  If you want to change these files, it is best to contact the OpenMadrigal
        development administrator (madrigal@haystack.mit.edu)
        """
        metadataFiles = ('metadata3/siteTab.txt', 'metadata/instTab.txt', 'metadata/instType.txt')
        url = 'http://cedar.openmadrigal.org/compareToArchive.py?filePath=%s&fileTextMd5=%s'
        
        for metadataFile in metadataFiles:
            localMetadataFile = os.path.join(self.__madDB.getMadroot(), 'metadata', os.path.basename(metadataFile))
            archivePath = 'madroot/%s' % (metadataFile)
            
            f = open(localMetadataFile)
            text = f.read().encode('utf-8')
            f.close()
            textMd5 = hashlib.md5(text)
            md5Str = textMd5.hexdigest() # md5 checksum of local metadata file

            thisUrl = url % (archivePath, md5Str)

            f = urllib.request.urlopen(thisUrl, timeout=200)
            result = f.read().decode('utf-8')
            f.close()

            items = result.split() # first item is latest revision tag, second is matching revision tag
            if len(items) != 2:
                raise IOError('Problem with url %s' % (thisUrl))

            if items[0] != 'None' and items[0] == items[1]:
                # everything is up to date
                continue

            if items[0] == 'None':
                # failed to find this metadata file
                raise IOError('Problem with url %s' % (thisUrl))

            if items[1] == 'None':
                # this metadata file must have been locally editted, print warning
                print('Metadata file %s has been locally edited - contact the OpenMadrigal administrator at madrigal@haystack.mit.edu to update central metadata' % (metadataFile))
                continue

            # this metadata file needs updating
            print('Downloading revised version of metadata file %s from OpenMadrigal' % (metadataFile))
            text = self.__openMad.getLatestSubversionVersion(os.path.join('madroot',metadataFile))
            f = open(localMetadataFile, 'w', encoding='utf-8')
            f.write(text)
            f.close()
            
    def _updateInstData(self):
        """_updateInstData updates the summary files instData.txt and instDataPriv.txt based on 
        latest version of expTabAll.txt
        
        instData.txt and instDataPriv.txt has three comma-delimited columns:
            1. siteId
            2. kinst of non-test experiments at that siteID (only if site has most data)
            3. space separated ordered list of years with data available
            
        instDataPriv.txt includes local private data, in addition to public.
        
        Note that a kinst will only listed at one site.  If there is local data, it will be listed as
        local.  If it listed at more than one non-local site, it will be listed at the one with more
        experiments.
        """
        summDict = {} # local dict with keys=siteID, value=dict with key=kinst, value = years with non-test data
        summDictPriv = {} # same as summDict, but includes local private data
        kinstDict = {} # local dict with keys = kinst, value = dict with key=siteID, value = number of
                       # experiments.  Used to print warning if inst at multiple sites and to decide which
                       # site to incliude
        madExpObj = madrigal.metadata.MadrigalExperiment(self.__madDB,
                                                         os.path.join(self.__madDB.getMadroot(),
                                                                      'metadata/expTabAll.txt'))
        
        archive_sites = set([8,10])
        localSiteID = self.__madDB.getSiteID()
        for i in range(madExpObj.getExpCount()):
            siteID = madExpObj.getExpSiteIdByPosition(i)
            security = madExpObj.getSecurityByPosition(i)
            url = madExpObj.getExpUrlByPosition(i)
            # skip test experiments
            if self.__madDB.isTestExperiment(url, siteID) and security == 0:
                continue
            # skip all non-local archived data
            if siteID != localSiteID and security in (2,3):
                continue
            kinst = madExpObj.getKinstByPosition(i)
            sDTList = madExpObj.getExpStartDateTimeByPosition(i)
            eDTList = madExpObj.getExpEndDateTimeByPosition(i)
            # create a year list
            yearList = list(range(sDTList[0], eDTList[0]+1))
            # add to summDictPriv if not already there
            if siteID not in summDictPriv:
                summDictPriv[siteID] = {}
            if kinst not in summDictPriv[siteID]:
                summDictPriv[siteID][kinst] = [] # empty list of years
            for thisYear in yearList:
                if thisYear not in summDictPriv[siteID][kinst]:
                    summDictPriv[siteID][kinst].append(thisYear)
            # add to summDict if not already there and not private
            if security in (0,2):
                if siteID not in summDict:
                    summDict[siteID] = {}
                if kinst not in summDict[siteID]:
                    summDict[siteID][kinst] = [] # empty list of years
                for thisYear in yearList:
                    if thisYear not in summDict[siteID][kinst]:
                        summDict[siteID][kinst].append(thisYear)
            # add to kinstDict
            if kinst not in kinstDict:
                kinstDict[kinst] = {}
            if siteID not in list(kinstDict[kinst].keys()):
                kinstDict[kinst][siteID] = 1
                kinstSet = set(kinstDict[kinst].keys())
                if len(kinstSet.difference(archive_sites)) > 1:
                    print(('Note: kinst %i found at multiple non-archive sites: %s' % (kinst, str(kinstSet.difference(archive_sites)))))
            else:
                kinstDict[kinst][siteID] += 1
                
        # write to output files
        delimiter = ' '
        outputNames = ('instData.txt', 'instDataPriv.txt')
        dictList = (summDict, summDictPriv)
        for i in range(len(outputNames)):
            f = open(os.path.join(self.__madDB.getMadroot(), 'metadata', outputNames[i]), 'w', encoding='utf-8')
            thisDict = dictList[i]
            siteIDKeys = list(thisDict.keys())
            siteIDKeys.sort()
            for siteID in siteIDKeys:
                kinstKeys = list(thisDict[siteID].keys())
                kinstKeys.sort()
                for kinst in kinstKeys:
                    
                    # verify this kinst/siteID combination is the desired one
                    if localSiteID != siteID: 
                        if localSiteID in list(kinstDict[kinst].keys()):
                            continue # this is not local site, but local site has data and local site always wins
                        if len(kinstDict[kinst]) > 1:
                            accept = True # test whether this non-local site has the most experiments
                            thisCount = kinstDict[kinst][siteID]
                            for thisKey in list(kinstDict[kinst].keys()):
                                if kinstDict[kinst][thisKey] > thisCount:
                                    accept = False
                                    break
                            if not accept:
                                continue
                            
                    # this data is accepted - write it out
                    yearsList = thisDict[siteID][kinst]
                    yearsList.sort()
                    yearsStrList = [str(year) for year in yearsList]
                    yearsStr = delimiter.join(yearsStrList)
                    f.write('%i,%i,%s\n' % (siteID, kinst, yearsStr))
                    
            f.close()
        
                

    def __walkExpDir__(self, arg, dirname, names):
        """__walkExpDir__ is a private method called by os.walk.  arg is a dict with keys:
        1. extText = text of combined expTab.txt to be appended to
        2. fileText = text of combined fileTab.txt to be appended to
        3. presentCount = total experiments done so far
        4. localSiteId = local site id (int)
        5. expIds =  a list of all expIds found in locals dirs - set earlier
        6. expIds2 = a list of all expIds used so far in summary expDir.txt
        
        Sets values in arg
        """
        if 'expTab.txt' not in names:
            return

        # defines allowed experiment directory names
        dirConvStr1 = '/experiments[0-9]*/[0-9][0-9][0-9][0-9]/[a-z][a-z0-9][a-z0-9]/[^/]*'

        # check that dirname follows rule experiments[0-9]*/YYYY/sss/*
        madroot = self.__madDB.getMadroot()
        if madroot[-1] == '/':
            madroot = madroot[:-1]
        startIndex = len(madroot)
        testDir = dirname[startIndex:]
        if re.match(dirConvStr1, testDir) == None:
            return
        
        # make sure we only descend four levels
        count = 0
        items = testDir.split('/')
        for item in items:
            if len(item) > 0:
                count += 1
        if count != 4:
            return
        try:
            
            expObj = madrigal.metadata.MadrigalExperiment(self.__madDB,
                                                          os.path.join(dirname, 'expTab.txt'))
    
            # skip it if security == -1 (ignore flag)
            if expObj.getSecurityByPosition(0) == -1:
                return
    
            # skip if wrong site id
            if expObj.getExpSiteIdByPosition(0) != arg['localSiteId']:
                print('Warning: Experiment %s has wrong site id = %i.  This site id = %i' % \
                      (dirname, expObj.getExpSiteIdByPosition(0), arg['localSiteId']))
                return
    
            # modify experiment id if needed
            thisExpId = expObj.getExpIdByPosition(0)
            isNewExpId = False
            if thisExpId <= self.__madDB.getSiteID() * 10000000 or thisExpId in arg['expIds2']:
                # find a new unique id
                maxId1 = numpy.max(numpy.array(arg['expIds']))
                maxId2 = numpy.max(numpy.array(arg['expIds2']))
                newExpId = int(max([maxId1,maxId2])) + 1
                    
                expObj.setExpIdByPosition(0, newExpId)
                arg['expText'].append(str(expObj))
                # sanity check
                if newExpId in arg['expIds2']:
                    raise ValueError('Duplicate id %i' % (newExpId))
                arg['expIds2'].append(newExpId)
                expObj.writeMetadata()
                isNewExpId = True
                
                print(('Updated metadata in %s' % (str(dirname))))
                
                
            elif thisExpId in arg['expIds2']:
                raise ValueError('found unexpected duplicate expId %i' % (thisExpId))
            
            else:
                # normal case - this id already set and not changing
                arg['expText'].append(str(expObj))
                # sanity check
                if thisExpId in arg['expIds2']:
                    raise ValueError('Duplicate id %i' % (thisExpId))
                arg['expIds2'].append(thisExpId)
            
            arg['presentCount'] += 1
    
            if 'fileTab.txt' not in names:
                print('Info: Experiment %s has no fileTab.txt' % (dirname))
                return
    
            fileObj = madrigal.metadata.MadrigalMetaFile(self.__madDB,
                                                         os.path.join(dirname, 'fileTab.txt'))
    
            # set expId for all files
            for i in range(fileObj.getFileCount()):
                if isNewExpId:
                    fileObj.setExpIdByPosition(i, newExpId)
                else:
                    fileObj.setExpIdByPosition(i, thisExpId)
                
    
            arg['fileText'].append(str(fileObj))
            
            if isNewExpId:
                fileObj.writeMetadata()
                
            if arg['presentCount'] % 1000 == 0:
                print(('Done %i out of %i exps' % (arg['presentCount'], arg['totalCount'])))
            
        except:
            print((' *** Exception encounted in experiment directory %s ***' % (dirname)))
            raise
        
        
    def __walkExpDirIds__(self, arg, dirname, names):
        """__walkExpDirIds__ is a private method called by os.walk.  arg is a dict with keys:
        1. extText = text of combined expTab.txt to be appended to
        2. fileText = text of combined fileTab.txt to be appended to
        3. presentCount = total experiments done so far
        4. localSiteId = local site id (int)
        5. expIds =  a list of all expIds found in locals dirs - to be set here
        6. expIds2 = a list of all expIds used so far in summary expDir.txt
        
        Sets values in arg
        """
        if 'expTab.txt' not in names:
            return

        # defines allowed experiment directory names
        dirConvStr1 = '/experiments[0-9]*/[0-9][0-9][0-9][0-9]/[a-z][a-z0-9][a-z0-9]/[^/]*'

        # check that dirname follows rule experiments[0-9]*/YYYY/sss/*
        madroot = self.__madDB.getMadroot()
        if madroot[-1] == '/':
            madroot = madroot[:-1]
        startIndex = len(madroot)
        testDir = dirname[startIndex:]
        if re.match(dirConvStr1, testDir) == None:
            return
        
        # make sure we only descend four levels
        count = 0
        items = testDir.split('/')
        for item in items:
            if len(item) > 0:
                count += 1
        if count != 4:
            return
        try:
            
            expObj = madrigal.metadata.MadrigalExperiment(self.__madDB,
                                                          os.path.join(dirname, 'expTab.txt'))
    
            # skip it if security == -1 (ignore flag)
            if expObj.getSecurityByPosition(0) == -1:
                return
    
            # skip if wrong site id
            if expObj.getExpSiteIdByPosition(0) != arg['localSiteId']:
                print('Warning: Experiment %s has wrong site id = %i.  This site id = %i' % \
                      (dirname, expObj.getExpSiteIdByPosition(0), arg['localSiteId']))
                return
    
            # add this id if unique
            thisExpId = expObj.getExpIdByPosition(0)
            if int(thisExpId) not in arg['expIds']:
                arg['expIds'].append(int(thisExpId))
                
            # increment totalCount
            arg['totalCount'] += 1
            
        except:
            print((' *** Exception encounted in experiment directory %s ***' % (dirname)))
            raise

        
        
        

class MadrigalNotify:
    """MadrigalNotify is an object used to send messages to an administrator about a Madrigal database.

    This object provides functions needed to send messages to an administrator about a Madrigal database, for now
    only sendAlert, which sends an email to the site administrator found is siteTab.txt (or if not
    possible, the admin in madrigal.cfg, and finally if all else fails, to root).

    Usage example:

        import madrigal.admin
    
        try:
        
            adminObj =  madrigal.admin.MadrigalNotify()
            adminObj.sendAlert('This is important!', 'Important Message')
            
        except madrigal.admin.MadrigalError, e:
        
            print e.getExceptionStr()


    Non-standard Python modules used:
    None

    Exceptions thrown: None - Note that MadrigalNotify tries every trick it knows to avoid
    throwing exceptions, since this is the class that will generally be called when there is a problem.

    Change history:

    Written by "Bill Rideout":mailto:wrideout@haystack.mit.edu  Dec. 4, 2001
    """
    

    #constants
    __defaultUser  = "root"
    """ Sets the default user to email to when all else fails. """

    __defaultServer  = "localhost"
    """ Sets the default server to send mail when all else fails. """


    def __init__(self, madDB = None):
        """__init__ initializes MadrigalNotify by getting some basic information from MadrigalDB and MadrigalSite.

        Note that MadrigalNotify tries every trick it knows to avoid throwing exceptions, since
        this is the class that will generally be called when there is a problem.

        Inputs: Existing MadrigalDB object, by default = None.
        
        Returns: void

        Affects: Initializes self.__binDir.

        Exceptions: None.
        """

        # get metadata dir
        if madDB == None:
            try:
                thisMadDB = madrigal.metadata.MadrigalDB()
            except:
                # note that the main configuration file is unavailable 
                # the best that can be done is send an email to root using localhost mailserver
                self.__emailAddress = self.__defaultUser
                self.__emailServer  = self.__defaultServer
                thisMadDB = None
        else:
            thisMadDB = madDB
        self.madDB = thisMadDB

        if thisMadDB != None:
            self.__emailServer  = thisMadDB.getMailserver()
            # now try to get email from site metadata, if failure, use config contact info
            try:
                thisSite = madrigal.metadata.MadrigalSite()
                self.__emailAddress = thisSite.getSiteEmail(thisMadDB.getSiteID())
                if self.__emailAddress == None:
                    # couldn't read metadata - use madrigal.cfg
                    self.__emailAddress = thisMadDB.getContactEmail()
            except:
                # couldn't read metadata - use madrigal.cfg
                self.__emailAddress = thisMadDB.getContactEmail()

            #make sure madrigal.cfg worked - if not use root
            if self.__emailAddress == None:
                self.__emailAddress = self.__defaultUser


    def sendAlert(self, message, subject = None):
        """sendAlert sends an email with the given message and optional title.

        Inputs: message (string), and optional title (string)
        
        Returns: void

        Affects: none

        Exceptions: None.
        """

        # set up message
        message = 'From: (Python Madrigal API) ' + self.__emailAddress + '\n' + \
        'To: ' + self.__emailAddress + '\n' + \
        'Subject: ' + str(subject) + '\n' + \
        'Content-type: text/html\n\n' + message

        server = smtplib.SMTP(self.__emailServer)
        server.sendmail(self.__emailAddress.split(',')[0],
                        self.__emailAddress.split(','), message)
        server.quit()
        
        
    def notify(self, email, message, subject):
        """notify sends an email with the given message and title to email.

        Inputs: email (string), message (string), and subject (string)
        
        Returns: void

        Affects: none

        Exceptions: None.
        """

        # set up message
        message = 'From: (Madrigal at %s) ' % (self.madDB.getMadServer()) + self.__emailAddress + '\n' + \
        'To: ' + email + '\n' + \
        'Subject: ' + str(subject) + '\n' + \
        'Content-type: text/html\n\n' + message

        server = smtplib.SMTP(self.__emailServer)
        server.sendmail(self.__emailAddress.split(',')[0],
                        email.split(','), message)
        server.quit()


class MadrigalError(Exception):
    """MadrigalError is an exception class that is thrown for all known errors in using Madrigal Py lib.

    Usage example:

        import sys, traceback
        import madrigal.admin
    
        try:
        
            test = open('ImportantFile.txt', 'r')
            
        except:
        
            raise madrigal.admin.MadrigalError('ImportantFile.txt not opened!',
                                                traceback.format_exception(sys.exc_info()[0],
                                                                        sys.exc_info()[1],
                                                                        sys.exc_info()[2]))
    """


    def __init__(self, strInterpretation, exceptionList):
        """ __init__ gathers the interpretation string along with all information from sys.exc_info().

        Inputs: strIntepretation - A string representing the programmer's interpretation of
        why the exception occurred

                exceptionList - a list of strings completely describing the exception.
                Generated by traceback.format_exception(sys.exc_info()[0],
                                                        sys.exc_info()[1],
                                                        sys.exc_info()[2])
        
        Returns: Void.

        Affects: Initializes class member variables _strInterp, _strExcList.

        Exceptions: None.
        """
        
        self._strInterp = strInterpretation
        self._strExcList = exceptionList

        
    def getExceptionStr(self):
        """ getExceptionStr returns a formatted string ready for printing completely describing the exception.

        Inputs: None
        
        Returns: A formatted string ready for printing completely describing the exception.

        Affects: None

        Exceptions: None.
        """
        excStr = 'The following Madrigal Python exception has occurred:\n'
        excStr = excStr + self._strInterp + '\n\n'

        if self._strExcList != None:
            for item in self._strExcList:
                excStr = excStr + str(item) + '\n'

        return excStr
    
    def __str__(self):
        return(self.getExceptionStr())


    def getExceptionHtml(self):
        """ getExceptionHtml returns an Html formatted string completely describing the exception.

        Inputs: None
        
        Returns: A formatted string ready for printing completely describing the exception.

        Affects: None

        Exceptions: None.
        """
        
        excStr = '
The following Madrigal Python exception has occurred:\n
' excStr = excStr + self._strInterp + '\n
\n' if self._strExcList != None: for item in self._strExcList: excStr = excStr + str(item) + '\n
' return excStr if __name__ == '__main__': test = MadrigalNotify() test.sendAlert('This is a message from the python module MadrigalNotify', 'Test from MadrigalNotify') print('Hopefully message sent - check.') test = MadrigalDBAdmin() """ expDir = test.createRTExperiment(datetime.datetime.now(), 4, 32, 'Dummy experiment', ('mlhrt1','mlhrt2'), (30,30), (0,0), ('preliminary','very preliminary')) mlhrtFile = open('/home/hyperion/brideout/mlhrt1') mlhrt = mlhrtFile.read() mlhrtFile.close() test.appendRTMadrigalFile(expDir, 'mlhrt1', mlhrt) # not real data, but shouldn't be checked test.appendRTMadrigalFile(expDir, 'mlhrt1', 'qqqqqqqqqqqqqqqqqqqqqqqq') mlhrtFile = open('/home/hyperion/brideout/mlhrt2') mlhrt = mlhrtFile.read() mlhrtFile.close() try: # this should raise a wrong file type exception test.appendRTMadrigalFile(expDir, 'mlhrt2', mlhrt) except: traceback.print_exc() expDir = test.createMadrigalExperiment('/home/hyperion/brideout/mlh050429c.000', 'Dummy experiment', 0, 'test exp', 30, 1) test.overwriteMadrigalFile(expDir, '/home/hyperion/brideout/junk/mlh050429c.000') test.addWebFile(expDir, '/home/hyperion/brideout/junk.html', 'html/second/')""" expDir = '/home/grail/brideout/madroot/experiments/1998/mlh/20jan98' try: test.addMadrigalFile(expDir, '/tmp/mlh980120g.001', 0, 'second file') except: traceback.print_exc() """ test.changeFileStatus(expDir, '/home/hyperion/brideout/mlh050429c.000', 3, 'new status', 1) print 'about to run updateMaster' test.updateMaster(True) """

Functions

def convertMad2FileToMad3(

mad2File, madDB=None, status=None)

convertMad2FileToMad3 will convert an input mad2File to Madrigal 3 HDF5. It returns the path to the newly creates file, which will be in the /tmp directory.

Inputs:

mad2File - A Madrigal 2 format file to convert

madDB - a madrigal.metadata.MadrigalDB object.  If None (the default) created.

status - if None, get status from fileTab.txt.  Else use status passed in

Returns: full path to newly created Madrigal 3 Hdf5 file. Basename is basename of input file, with .hdf5 extension

def convertMad2FileToMad3(mad2File, madDB=None, status=None):
    """convertMad2FileToMad3 will convert an input mad2File to Madrigal 3 HDF5.  It returns the path to the
    newly creates file, which will be in the /tmp directory.
    
    Inputs:
    
        mad2File - A Madrigal 2 format file to convert
        
        madDB - a madrigal.metadata.MadrigalDB object.  If None (the default) created.
        
        status - if None, get status from fileTab.txt.  Else use status passed in
        
    Returns: full path to newly created Madrigal 3 Hdf5 file.  Basename is basename of input file, with .hdf5 extension
    """
    if madDB is None:
        madDB = madrigal.metadata.MadrigalDB()
        
    # make sure an hdf5 file not passed in
    base, ext = os.path.splitext(mad2File)
    if ext in ('.hdf5', '.h5', '.hdf5'):
        raise ValueError('Cannot call convertMad2FileToMad3 with Hdf5 file %s' % (mad2File))
        
    madDataObj = madrigal.data.MadrigalFile(mad2File, madDB)
    kinst = madDataObj.getKinstList()[0]
    kindat = madDataObj.getKindatList()[0]
    
    extraParms, indParms, splitParms = madDB.getKinstKindatConfig(kinst, kindat)
    
    tmpFile = os.path.join('/tmp', os.path.basename(mad2File) + '.hdf5')
    # make sure it doesn't exist
    if os.access(tmpFile, os.R_OK):
        os.remove(tmpFile)
    
    madDataObj.exportToHdf(tmpFile, indParms, splitParms, extraParms, status=status)
    
    return(tmpFile)

Classes

class MadrigalDBAdmin

MadrigalDBAdmin is a class that allows modifications to be made to the Madrigal database

dbAdminObj = madrigal.admin.MadrigalDBAdmin()

expDir = dbAdminObj.createMadrigalExperiment('/home/hyperion/brideout/mlh050429c.000', 'Dummy experiment', 0, 'test exp', 30, 1)

Non-standard Python modules used: None

Change history:

Written by "Bill Rideout":mailto:wrideout@haystack.mit.edu May. 5, 2005

class MadrigalDBAdmin:
    """MadrigalDBAdmin is a class that allows modifications to be made to the Madrigal database

    dbAdminObj = madrigal.admin.MadrigalDBAdmin()

    expDir = dbAdminObj.createMadrigalExperiment('/home/hyperion/brideout/mlh050429c.000',
                                        'Dummy experiment',
                                        0,
                                        'test exp',
                                        30,
                                        1)

    Non-standard Python modules used: None

    Change history:

    Written by "Bill Rideout":mailto:wrideout@haystack.mit.edu  May. 5, 2005

    """
    
    def __init__(self, madDB = None):
        """__init__ initializes MadrigalDBAdmin
        
        Inputs: madDB - Existing MadrigalDB object.  Default = None.
        
        Returns: void

        Affects:

            Sets self.__madDB to MadrigalDB object
            Sets self.__madInst to MadrigalInstrument object
        """

        if madDB == None:
            self.__madDB = madrigal.metadata.MadrigalDB()
        else:
            self.__madDB = madDB

        self.__madInst = madrigal.metadata.MadrigalInstrument(self.__madDB)
        self.__openMad = madrigal.openmadrigal.OpenMadrigal(self.__madDB)
        self.__madSite = madrigal.metadata.MadrigalSite(self.__madDB)
        self._userData = madrigal.ui.userData.MadrigalUserData(self.__madDB)
        
            

    def createRTExperiment(self,
                           startTime,
                           numDays,
                           instrument,
                           expTitle,
                           rtFilenameList,
                           kindatList,
                           permissionList,
                           fileDescList,
                           optChar = '',
                           endTime = None,
                           security = 0,
                           dirName = None,
                           experimentsDirNum=None,
                           PI='', PIEmail='', 
                           fileAnalystList=None, fileAnalystEmailList=None,
                           notify = True):
        """createRTExperiment creates a new experiment on Madrigal in preparation for realtime data.

        Since the experiment is presumably not yet complete, metadata such as the duration of the experiment
        must be estimated.  This metadata will be overwritten when the first batch file is added.

        Inputs:
        
            startTime - experiment start time.  If a number, assumed to be seconds since 1/1/1970.  May also
            be a datetime.datetime object
            
            numDays - number of days the experiment is estimated to run.  Ignored if optional endTime given.
            
            instrument - instrument code or 3-letter Madrigal mnenonic
            
            expTitle - experiment title
            
            rtFilenameList - list of realtime filenames to be created. Must all be Hdf5 files.
            
            kindatList - list of ints of kindats for each realtime file.  Len = len(rtFilenameList)
            
            permissionList - list of 0 (public) or 1 (private). Len = len(rtFilenameList)
            
            fileDescList - list of realtime file descriptions

            optChar - optional character to be added to experiment directory if no dirName
                      given.  If dirName argument given, this argument ignored.  optChar
                      is used if the default directory name DDmmmYY is used for
                      more than one experiment created for a given instrument on a given day.
                      For example, if --optChar=h for a MLH experiment on September 12, 2005,
                      then the experiment directory created would be experiments/2005/mlh/12sep05h.

            endTime - optional end date and time of experiment.  If a number, assumed to be seconds since
            1/1/1970.  May also be a datetime.datetime object

            security - experiment security setting.  If 0 (the default) public.  If 1, private.
                       If -1, entire experiment ignored.  Any given file permission is the more
                       restricted of experiment permission and file permission.

            dirName - directory name to use for experiment.  If None (the default), the directory
                      name will be the default name DDmmmYY[optChar].  Cannot contain "/"
                      
            experimentsDirNum - the number to be appended to the experiments directory, if experiments
                      directory being used is of the form experiments[0-9]* instead of just
                      experiments.  For example, if experimentsDirNum is 7, then the experiment
                      would be created in MADROOT/experiments7 instead of MADROOT/experiments.
                      
            PI- full name of principal investigator.  The default is ''
            
            PIEmail - email of principal investigator.  The default is ''
            
            fileAnalystList - list of full names of file analysts, one for each file.  If None, the default, 
                File Analyst = ''
            
            fileAnalystEmailList - list of emails of file analysts, one for each file.  If None, the default, 
                File Analyst email = ''
                
            notify - if True (the default), send a message to all registered users.  If False, do not.

        Returns:

            Full path to directory created
            
        """
        # check optChar
        if type(optChar) not in (bytes, str):
            raise ValueError('optChar must be an empty or a one character string, not %s' % (str(optChar)))

        if len(optChar) > 1:
            raise ValueError('optChar must be an empty or a one character string, not %s' % (str(optChar)))

        security = int(security)
        if security not in (-1,0,1):
            raise ValueError('security must be -1, 0, or 1, not %i' % (security))
        
        # convert startTime to datetime if needed
        if type(startTime) in (int, int, float):
            startTime = datetime.datetime.utcfromtimestamp(startTime)

        # create endTime based on numDays
        if endTime != None:
            if type(endTime) in (int, int, float):
                endTime = datetime.datetime.utcfromtimestamp(startTime)
        else:
            if numDays >= 0:
                endTime = startTime + datetime.timedelta(numDays) - datetime.timedelta(0,1)
            else:
                raise ValueError('numDays must not be negative')

        if startTime >= endTime:
            raise ValueError('Experiment start time %s after end time %s' % (str(startTime),
                                                                               str(endTime)))
        

        # get instrument mnemonic and instCode
        try:
            instCode = int(instrument)
            instMnemonic = self.__madInst.getInstrumentMnemonic(instCode)
        except ValueError:
            if len(instrument) != 3:
                raise ValueError('%s not a legal instrument mnemonic' % (str(instrument)))
            instMnemonic = instrument.lower()
            # verify its a legal mnemonic
            instList = self.__madInst.getInstrumentList()
            found = False
            for inst in instList:
                if instMnemonic == inst[1]:
                    found = True
                    instCode = inst[2]
                    break
            if found == False:
                raise ValueError('%s not a legal instrument mnemonic or code' % (str(instrument)))

        if instMnemonic == None:
            raise ValueError('%s not a legal instrument mnemonic or code' % (str(instrument)))
        
        instDesc = self.__madInst.getInstrumentName(instCode)

        # expTitle
        if type(expTitle) not in (bytes, str) and expTitle != None:
            raise ValueError('expTitle not a string')
        if expTitle == None:
            expTitle = ''
        if expTitle.find(',') != -1:
            raise ValueError('expTitle cannot contain a comma')
            
        # PI
        if type(PI) not in (bytes, str) and PI != None:
            raise ValueError('PI not a string')
        if PI == None:
            PI = ''
        if PI.find(',') != -1:
            raise ValueError('PI cannot contain a comma')
            
        # PIEmail
        if type(PIEmail) not in (bytes, str) and PIEmail != None:
            raise ValueError('PIEmail not a string')
        if PIEmail == None:
            PIEmail = ''
        if PIEmail.find(',') != -1:
            raise ValueError('PIEmail cannot contain a comma')

        # rtFilenameList
        if type(rtFilenameList) not in (list, tuple):
            raise ValueError('rtFilenameList not a list or tuple')
        # make sure each rtFilename is a string without /
        for filename in rtFilenameList:
            if type(filename) not in (bytes, str):
                raise ValueError('rtFilenameList must contain strings')
            if filename.find('/') != -1:
                raise ValueError('rtFilenameList must contain strings without /')
            if filename.find(',') != -1:
                raise ValueError('filename cannot contain a comma')
            base, ext = os.path.splitext(filename)
            if ext not in ('.hdf5', '.h5', '.hdf5'):
                raise ValueError('All input files must have a valid Hdf5 extension, <%s> does not' % (filename))

        # kindatList
        if len(kindatList) != len(rtFilenameList):
            raise ValueError('length of kindatList not equal length of rtFilenameList')
        for item in kindatList:
            try:
                int(item)
            except:
                raise ValueError('kindatList must contain integers')

        # permissionList
        if len(permissionList) != len(rtFilenameList):
            raise ValueError('length of permissionList not equal length of rtFilenameList')
        for item in permissionList:
            try:
                permission = int(item)
            except:
                raise ValueError('permissionList must contain integers')

            if permission not in (0,1):
                raise ValueError('permissionList must contain integers of value 0 (public) or 1 (private)')

        
        # fileDescList
        if len(fileDescList) != len(rtFilenameList):
            raise ValueError('length of fileDescList not equal length of rtFilenameList')
        for item in fileDescList:
            if type(item) not in (bytes, str):
                raise ValueError('fileDescList must only contain strings')
            if item.find(',') != -1:
                raise ValueError('fileDesc cannot contain a comma')
            
        # fileAnalystList
        if fileAnalystList:
            if len(fileAnalystList) != len(rtFilenameList):
                raise ValueError('length of fileAnalystList not equal length of rtFilenameList')
            for item in fileAnalystList:
                if type(item) not in (bytes, str):
                    raise ValueError('fileAnalystList must only contain strings')
                if item.find(',') != -1:
                    raise ValueError('fileAnalyst cannot contain a comma')
                
        # fileAnalystEmailList
        if fileAnalystEmailList:
            if len(fileAnalystEmailList) != len(rtFilenameList):
                raise ValueError('length of fileAnalystEmailList not equal length of rtFilenameList')
            for item in fileAnalystEmailList:
                if type(item) not in (bytes, str):
                    raise ValueError('fileAnalystEmailList must only contain strings')
                if item.find(',') != -1:
                    raise ValueError('fileAnalystEmail cannot contain a comma')
            
        # experimentDirNum
        if experimentsDirNum != None:
            experimentsDir = 'experiments%i' % (experimentsDirNum)
            # verify this directory exists
            if not os.access(os.path.join(self.__madDB.getMadroot(), experimentsDir), os.R_OK):
                raise ValueError('no such directory %s' % (experimentsDir))
        else:
            experimentsDir = 'experiments'

        # all the arguments check out - create the directory
        # create experiment dir
        expDir2 = os.path.join('%i' % (startTime.year),
                               instMnemonic)
        
        if dirName == None:
            dirName = startTime.strftime('%d%b%y').lower() + optChar
        else:
            # verify dirName is basename
            if dirName.find('/') != -1:
                raise ValueError('dirName must be base directory name, not %s' % (dirName))

        expDir2 = os.path.join(expDir2, dirName)

        expDir = os.path.join(self.__madDB.getMadroot(), experimentsDir, expDir2)
        
        expDir3 = os.path.join(experimentsDir, expDir2)

        # if the directory already exists, raise error
        if os.access(expDir, os.R_OK):
            raise IOError('Directory %s already exists' % (expDir))

        os.makedirs(expDir)
        os.chmod(expDir, 0o775)

        # expTab.txt
        expTabText = '0,' + self.__madDB.getTopLevelUrl()
        if expTabText[-1] != '/':
            expTabText += '/madtoc/'
        else:
            expTabText += 'madtoc/'
        expTabText += '%s,%s,%i,%04i%02i%02i,%02i%02i%02i,%04i%02i%02i,%02i%02i%02i,%i,%i,%s,%s\n' %(expDir3,
                                                                                                     expTitle,
                                                                                                     self.__madDB.getSiteID(),
                                                                                                     startTime.year,
                                                                                                     startTime.month,
                                                                                                     startTime.day,
                                                                                                     startTime.hour,
                                                                                                     startTime.minute,
                                                                                                     startTime.second,
                                                                                                     endTime.year,
                                                                                                     endTime.month,
                                                                                                     endTime.day,
                                                                                                     endTime.hour,
                                                                                                     endTime.minute,
                                                                                                     endTime.second,
                                                                                                     instCode,
                                                                                                     security,
                                                                                                     PI,
                                                                                                     PIEmail)

        # write expTab.txt
        f = open(os.path.join(expDir, 'expTab.txt'), 'w', encoding='utf-8')
        f.write(expTabText)
        f.close()
        os.chmod(os.path.join(expDir, 'expTab.txt'), 0o664)

        # fileTab.txt
        # set file times to right now UT
        nowUT = datetime.datetime.utcnow()
        nowDate = nowUT.strftime('%Y%m%d')
        nowTime = nowUT.strftime('%H%M%S')
        fileTabText = ''
        for index in range(len(rtFilenameList)):
            
            if fileAnalystList and fileAnalystEmailList:
                fileAnalyst = fileAnalystList[index]
                fileAnalystEmail = fileAnalystEmailList[index]
            else:
                fileAnalyst = ''
                fileAnalystEmail = ''
            
            fileTabText += rtFilenameList[index]
            fileTabText += ',0,%i,4,0,0,0,%s,%s,' % (kindatList[index], nowDate, nowTime)
            fileTabText += '%s,%i,%s,%s\n' % (fileDescList[index], permissionList[index],
                                              fileAnalyst, fileAnalystEmail)

        # write fileTab.txt
        f = open(os.path.join(expDir, 'fileTab.txt'), 'w')
        f.write(fileTabText)
        f.close()
        os.chmod(os.path.join(expDir, 'fileTab.txt'), 0o664)

        # create all writeable directory overview
        os.makedirs(os.path.join(expDir, 'overview'))
        os.chmod(os.path.join(expDir, 'overview'), 0o777)
        
        if notify:
            # get expPath without MAD ROOT
            expPath = expDir[expDir.find('experiments'):]
            if expPath[-1] == '/':
                expPath = expPath[:-1]
            madNotify = MadrigalNotify(self.__madDB)
            message = 'You requested to be notified when new data from instrument %s was available. A new realtime experiment for that instrument has been created at %s at %s. '   \
                % (instDesc, self.__madDB.getTopLevelUrl(), str(datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S')))
            message += 'Please contact %s if you wish to be unregistered.' %  (self.__madDB.getContactEmail())
                
            # deal with users registered for instrument in general
            userList = self._userData.getRegisteredInstUsers(instCode)
            if len(userList) > 0:
                for user in userList:
                    try:
                        madNotify.notify(user, message, 'Update to Madrigal instrument you registered interest in')
                    except:
                        print(('Warning - email to %s failed to be sent' % (user)))

        return(expDir)
        
        
    def writeRTMadrigalFile(self,
                            expDir,
                            rtFilename,
                            rtFile):
        """writeRTMadrigalFile writes a realtime Madrigal file to a Madrigal experiment directory.

        Fails if rtFilename does not match one listed in fileTab.txt.

        Inputs:
        
            expDir - full path to experiment directory (as returned by createRTExperiment)
            
            rtFilename - basename of realtime file to be writtem
            
            rtFile - a string containing the realtime file contents

        Returns: None

        Raises exception if rtFilename does not match one listed in fileTab.txt.
        """
        # verify rtFilename listed in fileTab.txt
        try:
            fileInfo = madrigal.metadata.MadrigalMetaFile(self.__madDB, os.path.join(expDir, 'fileTab.txt'))
        except:
            raise ValueError('Unable to open fileTab.txt in %s' % (expDir))
        
        if fileInfo.getHasCatalogByFilename(rtFilename) == None:
            raise ValueError('Filename %s not found in fileTab.txt' % (rtFilename))

        # okay - write it
        f = open(os.path.join(expDir, rtFilename), 'w', encoding='utf-8')
        f.write(rtFile)
        f.close()
        os.chmod(os.path.join(expDir, rtFilename), 0o664)


    def appendRTMadrigalFile(self,
                             expDir,
                             rtFilename,
                             rtFile):
        """appendRTMadrigalFile used to allow appending to a realtime Madrigal file. No longer possible with Hdf5 format.

        Inputs:
        
            expDir - full path to experiment directory (as returned by createRTExperiment)
            
            rtFilename - basename of realtime file to be writtem
            
            rtFile - a string containing the new realtime file contents

        Always raises IOError
        """
        raise IOError('The method appendRTMadrigalFile no longer supported by Madrigal 3.0')
        


    def createMadrigalExperiment(self,
                                 madFilename,
                                 expTitle,
                                 permission,
                                 fileDesc,
                                 instCode = None,
                                 category = 1,
                                 optChar = '',
                                 dirName = None,
                                 kindat = None,
                                 experimentsDirNum=None,
                                 PI='', PIEmail='', 
                                 fileAnalyst='', fileAnalystEmail='',
                                 createCachedText=False, createCachedNetCDF4=False,
                                 notify = True, updateToMad3=False):
        """createMadrigalExperiment creates a new experiment on Madrigal using metadata read from madFilename.

        Inputs:
        
            madFilename - full path to the complete Madrigal file.  Basename will be maintained.
            
            expTitle - experiment title
            
            permission - 0 (public) or 1 (private) or -1 (ignore). 
            
            fileDesc - file description

            instCode - instrument code.  If default (None), instrument code is taken from file, but error
            is thrown if more than one kinst found.

            category - 1=default, 2=variant, 3=history, or 4=realtime. Default is 1 (default file)

            optChar - optional character to be added to experiment directory if no dirName
                      given.  If dirName argument given, this argument ignored.  optChar
                      is used if the default directory name DDmmmYY is used for
                      more than one experiment created for a given instrument on a given day.
                      For example, if --optChar=h for a MLH experiment on September 12, 2005,
                      then the experiment directory created would be experiments/2005/mlh/12sep05h.

            dirName - directory name to use for experiment.  If None (the default), the directory
                      name will be the default name DDmmmYY[optChar].  Cannot contain "/"

            kindat - if not None (the default), use this kindat instead of what is found in the file.
            
            experimentsDirNum - the number to be appended to the experiments directory, if experiments
                      directory being used is of the form experiments[0-9]* instead of just
                      experiments.  For example, if experimentsDirNum is 7, then the experiment
                      would be created in MADROOT/experiments7 instead of MADROOT/experiments.
                      
            PI- full name of principal investigator.  The default is ''
            
            PIEmail - email of principal investigator.  The default is ''
            
            fileAnalyst -full name of file analyst.  The default is ''
            
            fileAnalystEmail - email of file analyst,.  The default is ''
            
            createCachedText - if True, add cached text file in overview/.txt.gz.  If False,
                no cached file.
                
            createCachedNetCDF4 - if True, add cached netCDF4 file in overview/.nc.  If False,
                no cached file.
                
            notify - if True (the default), send a message to all registered users.  If False, do not.
            
            updateToMad3 - if False (the default), error raised if madFilename non-Hdf5 file. If True, try to
                convert madFilename to Madrigal with .hdf5 extension before loading.
            
        Returns:

            Full path to directory created
            
        """
        # check optChar
        if type(optChar) not in (bytes,str):
            raise ValueError('optChar must be an empty or a one character string, not %s' % (str(optChar)))

        if len(optChar) > 1:
            raise ValueError('optChar must be an empty or a one character string, not %s' % (str(optChar)))
        
        base, ext = os.path.splitext(madFilename)
        if updateToMad3:
            if ext not in ('.hdf5', '.h5', '.hdf5'):
                madFilename = convertMad2FileToMad3(madFilename, self.__madDB, fileDesc)
        elif ext not in ('.hdf5', '.h5', '.hdf5'):
            raise ValueError('called createMadrigalExperiment with non-Hdf5 file <%s> and updateToMad3 False' \
                % (madFilename))
                
        fileInfo = madrigal.data.MadrigalFile(madFilename, self.__madDB)

        # get startTime
        sTime = fileInfo.getEarliestTime()
        startTime = datetime.datetime(sTime[0],sTime[1],sTime[2],sTime[3],sTime[4],sTime[5])

        # get endTime
        eTime = fileInfo.getLatestTime()
        endTime = datetime.datetime(eTime[0],eTime[1],eTime[2],eTime[3],eTime[4],eTime[5])
        
        # get instrument mnemonic and instCode
        if instCode == None:
            kinstList = fileInfo.getKinstList()
            if len(kinstList) == 0:
                raise ValueError('No kinst values found in file')
            if len(kinstList) > 1:
                raise ValueError('More than one kinst value found in file: %s' % (str(kinstList)))
            instCode = kinstList[0]
        instMnemonic = self.__madInst.getInstrumentMnemonic(instCode)
        instDesc = self.__madInst.getInstrumentName(instCode)
        if instMnemonic == None:
            raise ValueError('Unable to find mnemonic for kinst %i' % (instCode))

        # expTitle
        if type(expTitle) not in (bytes, str) and expTitle != None:
            raise ValueError('expTitle not a string')
        if expTitle == None:
            expTitle = ''
        if expTitle.find(',') != -1:
            raise ValueError('expTitle cannot contain a comma')

        # kindat
        if kindat == None:
            kindatList = fileInfo.getKindatList()
            if len(kindatList) == 0:
                raise ValueError('No kindat values found in file')
            if len(kindatList) > 1:
                raise ValueError('More than one kindat value found in file: %s' % (str(kindatList)))
            kindat = kindatList[0]
        else:
            kindat = int(kindat)

        # permission
        if permission not in (0,1, -1):
            raise ValueError('permission must be either 0 or 1 or -1, not %s' % (str(permission)))

        # fileDesc
        if type(fileDesc) not in  (bytes, str) and fileDesc != None:
            raise ValueError('fileDesc not a string')
        if fileDesc == None:
            fileDesc = ''
        if fileDesc.find(',') != -1:
            raise ValueError('fileDesc cannot contain a comma')

        # category
        if category not in (1,2,3,4):
            raise ValueError('category must be 1=default, 2=variant, 3=history, or 4=realtime; not %s' % (str(category)))

        # hasCatalog and hasHeader
        catStr = fileInfo.getCatalogHeaderStr()
        if len(catStr) > 0:
            hasCatalog = 1
            hasHeader = 1
        else:
            hasCatalog = 0
            hasHeader = 0
            
        # experimentDirNum
        if experimentsDirNum != None:
            experimentsDir = 'experiments%i' % (experimentsDirNum)
            # verify this directory exists
            if not os.access(os.path.join(self.__madDB.getMadroot(), experimentsDir), os.R_OK):
                raise ValueError('no such directory %s' % (experimentsDir))
        else:
            experimentsDir = 'experiments'
            
        # PI
        if type(PI) not in (bytes, str) and PI != None:
            raise ValueError('PI not a string')
        if PI == None:
            PI = ''
        if PI.find(',') != -1:
            raise ValueError('PI cannot contain a comma')
            
        # PIEmail
        if type(PIEmail) not in (bytes, str) and PIEmail != None:
            raise ValueError('PIEmail not a string')
        if PIEmail == None:
            PIEmail = ''
        if PIEmail.find(',') != -1:
            raise ValueError('PIEmail cannot contain a comma')
            
        # fileAnalyst
        if type(fileAnalyst) not in (bytes, str) and fileAnalyst != None:
            raise ValueError('fileAnalyst not a string')
        if fileAnalyst == None:
            fileAnalyst = ''
        if fileAnalyst.find(',') != -1:
            raise ValueError('fileAnalyst cannot contain a comma')
            
        # fileAnalystEmail
        if type(fileAnalystEmail) not in (bytes, str) and fileAnalystEmail != None:
            raise ValueError('fileAnalystEmail not a string')
        if fileAnalystEmail == None:
            fileAnalystEmail = ''
        if fileAnalystEmail.find(',') != -1:
            raise ValueError('fileAnalystEmail cannot contain a comma')
        
        # all the arguments check out - create the directory
        # create experiment dir
        expDir2 = os.path.join('%i' % (startTime.year),
                              instMnemonic)
        
        if dirName == None:
            dirName = startTime.strftime('%d%b%y').lower() + optChar
        else:
            # verify dirName is basename
            if dirName.find('/') != -1:
                raise ValueError('dirName must be base directory name, not %s' % (dirName))
            
        expDir2 = os.path.join(expDir2, dirName)

        expDir = os.path.join(self.__madDB.getMadroot(), experimentsDir, expDir2)
        
        expDir3 = os.path.join(experimentsDir, expDir2)

        # if the directory already exists, raise error
        if os.access(expDir, os.R_OK):
            raise IOError('Directory %s already exists' % (expDir))

        os.makedirs(expDir)
        os.chmod(expDir, 0o775)

        # expTab.txt
        expTabText = '0,' + self.__madDB.getTopLevelUrl()
        if expTabText[-1] != '/':
            expTabText += '/madtoc/'
        else:
            expTabText += 'madtoc/'
        expTabText += '%s,%s,%i,%04i%02i%02i,%02i%02i%02i,%04i%02i%02i,%02i%02i%02i,%i,%i,%s,%s\n' %(expDir3,
                                                                                                     expTitle,
                                                                                                     self.__madDB.getSiteID(),
                                                                                                     startTime.year,
                                                                                                     startTime.month,
                                                                                                     startTime.day,
                                                                                                     startTime.hour,
                                                                                                     startTime.minute,
                                                                                                     startTime.second,
                                                                                                     endTime.year,
                                                                                                     endTime.month,
                                                                                                     endTime.day,
                                                                                                     endTime.hour,
                                                                                                     endTime.minute,
                                                                                                     endTime.second,
                                                                                                     instCode,
                                                                                                     permission,
                                                                                                     PI, PIEmail)

        # write expTab.txt
        f = open(os.path.join(expDir, 'expTab.txt'), 'w', encoding='utf-8')
        f.write(expTabText)
        f.close()
        os.chmod(os.path.join(expDir, 'expTab.txt'), 0o664)

        # fileTab.txt
         # set file times to right now UT
        nowUT = datetime.datetime.utcnow()
        nowDate = nowUT.strftime('%Y%m%d')
        nowTime = nowUT.strftime('%H%M%S')
        fileTabText = os.path.basename(madFilename)
        fileTabText += ',0,%i,%i,0,%i,%i,%s,%s,' % (kindat,category,hasCatalog,hasHeader,nowDate,nowTime)
        fileTabText += '%s,%i,%s,%s\n' % (fileDesc, permission,
                                          fileAnalyst, fileAnalystEmail)

        # write fileTab.txt
        f = open(os.path.join(expDir, 'fileTab.txt'), 'w', encoding='utf-8')
        f.write(fileTabText)
        f.close()
        os.chmod(os.path.join(expDir, 'fileTab.txt'), 0o664)

        # create all writeable directory overview
        os.makedirs(os.path.join(expDir, 'overview'))
        os.chmod(os.path.join(expDir, 'overview'), 0o777)

        # cp madFilename to new directory
        shutil.copy2(madFilename, os.path.join(expDir, os.path.basename(madFilename)))
        os.chmod(os.path.join(expDir, os.path.basename(madFilename)), 0o664)

        # populate overview
        overviewFile = os.path.join(os.path.dirname(madFilename), 'overview', os.path.basename(madFilename) + '.summary')
        if os.access(overviewFile, os.R_OK):
            # make sure overview exist
            overviewDir = os.path.join(expDir, 'overview')
            if not os.access(overviewDir, os.W_OK):
                os.mkdir(overviewDir)
                os.chmod(overviewDir, 0o777)
            shutil.copy2(overviewFile, overviewDir)
        else:
            fileInfo = madrigal.data.MadrigalFile(os.path.join(expDir, os.path.basename(madFilename)), self.__madDB,
                                                  acceptOldSummary=acceptOldSummary)
        
        if createCachedText:
            cachedName = os.path.join(expDir, 'overview', os.path.basename(madFilename) + '.txt')
            madrigal.cedar.convertToText(madFilename, cachedName)
            subprocess.check_call(['gzip', cachedName])
            
        if createCachedNetCDF4:
            cachedName = os.path.join(expDir, 'overview', os.path.basename(madFilename) + '.nc')
            madrigal.cedar.convertToNetCDF4(madFilename, cachedName)
        
        if notify:
            # get expPath without MAD ROOT
            expPath = expDir[expDir.find('experiments'):]
            if expPath[-1] == '/':
                expPath = expPath[:-1]
            madNotify = MadrigalNotify(self.__madDB)
            message = 'You requested to be notified when the instrument %s from the Madrigal site %s was updated. A new experiment at %s was created on %s.'  \
                % (instDesc, self.__madDB.getTopLevelUrl(), expDir3, str(datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S')))
            message += 'Please contact %s if you wish to be unregistered.' % (self.__madDB.getContactEmail())
                
            # deal with users registered for instrument in general
            userList = self._userData.getRegisteredInstUsers(instCode)
            if len(userList) > 0:
                for user in userList:
                    try:
                        madNotify.notify(user, message, 'Update to Madrigal instrument data you registered interest in')
                    except:
                        print(('Warning - email to %s failed to be sent' % (user)))

        return(expDir)



    def changeExpStatus(self,
                        expDir,
                        expUrl=None,
                        expName = None,
                        siteID = None,
                        startDatetime = None,
                        endDatetime = None,
                        inst = None,
                        security = None,
                        PI = None,
                        PIEmail = None):
        """changeExpStatus is used to change attributes in expTab.txt.  If None, no change.

        Inputs:
        
            expDir - full path to experiment directory. Required.  Example:
               "/opt/madrigal/experiments/1998/mlh/20jan98". If None, do not change.
            
            expUrl - must be in form /madtoc/YYYY/<3 letter lower case inst code>/
                       example: http://www.haystack.mit.edu/cgi-bin/madtoc/1997/mlh/03dec97g.
                       If None, do not change.

            expName - experiment name.  Quotes required if contains spaces.  Example: "World Day"
                        If None, do not change.

            siteID - Madrigal siteID (int) of where data will be stored.  Error raised if not the siteID
                       of the local Madrigal site. Example: 4. If None, do not change.

            startDatetime - new start datetime of experiment (UT). If None, do not change.

            endDatetime - new end datetime of experiment (UT). If None, do not change.

            inst - new instrument code (int).  Example: 30. If None, do not change.  Prints
                    warning if not found in instTab.txt

            security - new security code.  Allowed values are 0 for public, 1 for private (limited IP range access)
                    -1 for ignore, 2 for archived experiment, 3 for private (limited IP range access) archived
                    experiment. If None, do not change.
                    
            PI - name of PI. If None, no change
            
            PIEmail - PI email.  If None, no change
            
        """
        try:
            expTabInfo = madrigal.metadata.MadrigalExperiment(self.__madDB, os.path.join(expDir, 'expTab.txt'))
        except:
            raise ValueError('Unable to open expTab.txt in %s' % (expDir))

        # be sure only one experiment
        if expTabInfo.getExpCount() != 1:
            raise ValueError('expTab.txt in %s has %i experiments, should have exactly 1' % (expDir,
                                                                                              expTabInfo.getExpCount()))

        # expUrl
        if expUrl != None:
            # print warning if not this directory
            index = expUrl.find('/madtoc/')
            thisDir = expUrl[index+8:]
            if thisDir[-1] == '/':
                thisDir = thisDir[:-1]
            if expDir.find(thisDir) == -1:
                raise ValueError('The experiment url you are setting this experiment to <%s> conflicts with experiment directory %s' % (expUrl, expDir))
            expTabInfo.setExpUrlByPosition(0, expUrl)

        # expName
        if expName != None:
            expTabInfo.setExpNameByPosition(0, expName)

        # siteID
        if siteID != None:
            siteID = int(siteID)
            if siteID != self.__madDB.getSiteID():
                raise ValueError('Setting experiment to a siteID %i different from this site\'s id %i' % (siteID,
                                                                                                                    self.__madDB.getSiteID()))
            expTabInfo.setExpSiteIdByPosition(0, siteID)

        # startDatetime
        if startDatetime != None:
            # verify before endDatetime if that also being set
            if endDatetime != None:
                if startDatetime > endDatetime:
                    raise ValueError('startDatetime %s must be before endDatetime %s' % (str(startDatetime),
                                                                                          str(endDatetime)))
            expTabInfo.setExpStartDateTimeByPosition(startDatetime, 0)

        # endDatetime
        if endDatetime != None:
            expTabInfo.setExpEndDateTimeByPosition(endDatetime, 0)

        # inst
        if inst != None:
            inst = int(inst)
            if self.__madInst.getInstrumentName(inst) == None:
                print('WARNING: instrument %i not found in instTab.txt' % (inst))
            expTabInfo.setExpKinstByPosition(0, inst)

        # security
        if security != None:
            security = int(security)
            if security not in (-1, 0, 1, 2, 3):
                raise ValueError('security must be in (-1, 0, 1, 2, 3), not %i' % (inst))
            expTabInfo.setSecurityByPosition(0, security)
            
        # PI
        if PI != None:
            expTabInfo.setPIByPosition(0, PI)
            
        # PIEmail
        if PIEmail != None:
            expTabInfo.setPIEmailByPosition(0, PIEmail)

        # everything successfully changed - write new values
        expTabInfo.writeMetadata()
        
            

    def addMadrigalFile(self,
                        expDir,
                        madFilename,
                        permission,
                        fileDesc,
                        category = 1,
                        kindat = None,
                        notify = True,
                        fileAnalyst = '',
                        fileAnalystEmail = '',
                        createCachedText=False, createCachedNetCDF4=False,
                        updateToMad3=False, acceptOldSummary=False):
        """addMadrigalFile adds a new file to an experiment using metadata read from madFilename.

        Inputs:

            expDir - full path to experiment directory (as returned by createMadriogalExperiment)
        
            madFilename - full path to the complete Madrigal file.  Basename will be maintained.
            
            permission - 0 (public) or 1 (private). 
            
            fileDesc - file description

            category - 1=default, 2=variant, 3=history, or 4=realtime. Default is 1 (default file)

            kindat - if not None (the default), use this kindat instead of what is found in the file.
            
            notify - if True (the default), send a message to all registered users.  If False, do not.
            
            fileAnalyst - full name of file Analyst.  Default is ''
            
            fileAnalystEmail - email of file Analyst.  Default is ''
            
            createCachedText - if True, add cached text file in overview/.txt.gz.  If False,
                no cached file.
                
            createCachedNetCDF4 - if True, add cached netCDF4 file in overview/.nc.  If False,
                no cached file.
              
            updateToMad3 - if False (the default), error raised if madFilename non-Hdf5 file. If True, try to
                convert madFilename to Madrigal with .hdf5 extension before loading.
                
            acceptOldSummary - if True, accept an old summary file. Used mainly for upgrading to Madrigal 3. Default
                is False.

        Returns: None
            
        """
        base, ext = os.path.splitext(madFilename)
        if updateToMad3:
            if ext not in ('.hdf5', '.h5', '.hdf5'):
                madFilename = convertMad2FileToMad3(madFilename, self.__madDB, fileDesc)
        elif ext not in ('.hdf5', '.h5', '.hdf5'):
            raise ValueError('called addMadrigalFile with non-Hdf5 file <%s> and updateToMad3 False' \
                % (madFilename))
                
        fileInfo = madrigal.data.MadrigalFile(madFilename, self.__madDB, acceptOldSummary=acceptOldSummary)

        # kindat
        if kindat == None:
            kindatList = fileInfo.getKindatList()
            if len(kindatList) == 0:
                raise ValueError('No kindat values found in file')
            if len(kindatList) > 1:
                raise ValueError('More than one kindat value found in file: %s' % (str(kindatList)))
            kindat = kindatList[0]
        else:
            kindat = int(kindat)

        # permission
        if permission not in (0,1):
            raise ValueError('permission must be either 0 or 1, not %s' % (str(permission)))

        # fileDesc
        if type(fileDesc) not in (bytes, str) and fileDesc != None:
            raise ValueError('fileDesc not a string')
        if fileDesc == None:
            fileDesc = ''
        # check that fileDesc does not illegally contain a comma
        if fileDesc.find(',') != -1:
            raise ValueError('fileDesc string in fileTab.txt cannot contain a comma: <%s> is illegal' % (fileDesc))

        # category
        if category not in (1,2,3,4):
            raise ValueError('category must be 1=default, 2=variant, 3=history, or 4=realtime; not %s' % (str(category)))

        # hasCatalog and hasHeader
        catStr = fileInfo.getCatalogHeaderStr()
        if len(catStr) > 0:
            hasCatalog = 1
            hasHeader = 0
        else:
            hasCatalog = 0
            hasHeader = 0
            
        if fileAnalyst.find(',') != -1:
            raise ValueError('fileAnalyst cannot contain a comma')
        
        if fileAnalystEmail.find(',') != -1:
            raise ValueError('fileAnalystEmail cannot contain a comma')
        

        # all the arguments check out - add line to fileTab.txt
        if not os.access(os.path.join(expDir, 'fileTab.txt'), os.R_OK):
            raise ValueError(' file %s does not yet exist' % (os.path.join(expDir, 'fileTab.txt')))

        # check that this is a new filename
        fileTabObj = madrigal.metadata.MadrigalMetaFile(self.__madDB, os.path.join(expDir, 'fileTab.txt'))
        for i in range(fileTabObj.getFileCount()):
            filename = fileTabObj.getFilenameByPosition(i)
            if filename == os.path.basename(madFilename):
                raise ValueError('File %s already exists - must be deleted first' % (filename))
            

        # fileTab.txt
        nowUT = datetime.datetime.utcnow()
        nowDate = nowUT.strftime('%Y%m%d')
        nowTime = nowUT.strftime('%H%M%S')
        fileTabText = os.path.basename(madFilename)
        fileTabText += ',0,%i,%i,0,%i,%i,%s,%s,' % (kindat,category,hasCatalog,hasHeader,nowDate,nowTime)
        fileTabText += '%s,%i,%s,%s\n' % (fileDesc, permission,
                                          fileAnalyst, fileAnalystEmail)

        # write fileTab.txt
        f = open(os.path.join(expDir, 'fileTab.txt'), 'a')
        f.write(fileTabText)
        f.close()                                                                                                                

        # cp madFilename to new directory
        shutil.copy2(madFilename, os.path.join(expDir, os.path.basename(madFilename)))
        os.chmod(os.path.join(expDir, os.path.basename(madFilename)), 0o664)

        # populate overview
        overviewFile = os.path.join(os.path.dirname(madFilename), 'overview', os.path.basename(madFilename) + '.summary')
        if os.access(overviewFile, os.R_OK):
            # make sure overview exist
            overviewDir = os.path.join(expDir, 'overview')
            if not os.access(overviewDir, os.W_OK):
                os.mkdir(overviewDir)
                os.chmod(overviewDir, 0o777)
            shutil.copy2(overviewFile, overviewDir)
        else:
            fileInfo = madrigal.data.MadrigalFile(os.path.join(expDir, os.path.basename(madFilename)), self.__madDB,
                                                  acceptOldSummary=acceptOldSummary)
                
        if createCachedText:
            cachedName = os.path.join(expDir, 'overview', os.path.basename(madFilename) + '.txt')
            madrigal.cedar.convertToText(madFilename, cachedName)
            subprocess.check_call(['gzip', cachedName])
            
        if createCachedNetCDF4:
            cachedName = os.path.join(expDir, 'overview', os.path.basename(madFilename) + '.nc')
            madrigal.cedar.convertToNetCDF4(madFilename, cachedName)

        # update expTab.txt against all registered files
        self.updateExpTab(expDir)
        
        
        if notify:
            # get expPath without MAD ROOT
            expPath = expDir[expDir.find('experiments'):]
            if expPath[-1] == '/':
                expPath = expPath[:-1]
            madNotify = MadrigalNotify(self.__madDB)
            message = 'You requested to be notified when the experiment: %s from the Madrigal site %s was updated. A new file %s has been added to that experiment at %s. '  \
                % (expPath, self.__madDB.getTopLevelUrl(), os.path.basename(madFilename), str(datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S')))
            message += 'Please contact %s if you wish to be unregistered.' % (self.__madDB.getContactEmail())
                
            # first deal with users registered for just this experiment
            userList = self._userData.getRegisteredUsers(expPath)
            if len(userList) > 0:
                for user in userList:
                    try:
                        madNotify.notify(user, message, 'Update to Madrigal experiment you registered interest in')
                    except:
                        print(('Warning - email to %s failed to be sent' % (user)))
                        
            # next deal with users registered for instrument in general
            madExpObj = madrigal.metadata.MadrigalExperiment(self.__madDB, os.path.join(expDir, 'expTab.txt'))
            kinst = madExpObj.getKinstByPosition(0)
            userList = self._userData.getRegisteredInstUsers(kinst)
            if len(userList) > 0:
                for user in userList:
                    try:
                        madNotify.notify(user, message, 'Update to Madrigal instrument data you registered interest in')
                    except:
                        print(('Warning - email to %s failed to be sent' % (user)))
            



    def changeFileStatus(self,
                         expDir,
                         filename,
                         category = None,
                         fileDesc = None,
                         permission = None,
                         kindat = None,
                         fileAnalyst = None,
                         fileAnalystEmail = None):
        """changeFileStatus is used to change category, fileDesc, or permission of a register file in fileTab.txt.

        Inputs:

            expDir - full path to experiment directory 
        
            filename - basename of existing Madrigal file already registered in fileTab.txt
            
            permission - 0 (public) or 1 (private). If None (default), leave unchanged. 
            
            fileDesc - file description. If None (default), leave unchanged.

            category - 1=default, 2=variant, or 3=history. If None (default), leave unchanged.

            kindat - kindat (int). If None (default), leave unchanged.
            
            fileAnalyst - name of file analyst.  If None (default), leave unchanged.
            
            fileAnalystEmail - email of file analyst.  If None (default), leave unchanged.
        """
        try:
            fileTabInfo = madrigal.metadata.MadrigalMetaFile(self.__madDB, os.path.join(expDir, 'fileTab.txt'))
        except:
            raise ValueError('Unable to open fileTab.txt in %s' % (expDir))

        # be sure filename is basename
        filename = os.path.basename(filename)

        # search all Madrigal files in fileTab.txt for right file
        found = False
        fileCount = fileTabInfo.getFileCount()

        for index in range(fileCount):
            thisFilename = fileTabInfo.getFilenameByPosition(index)
            if thisFilename == filename:
                found = True
                if category != None:
                    fileTabInfo.setCategoryByPosition(index, category)
                if fileDesc != None:
                    fileTabInfo.setStatusByPosition(index, fileDesc)
                if permission != None:
                    fileTabInfo.setAccessByPosition(index, permission)
                if kindat != None:
                    fileTabInfo.setKindatByPosition(index, kindat)
                if fileAnalyst != None:
                    fileTabInfo.setAnalystByPosition(index, fileAnalyst)
                if fileAnalystEmail != None:
                    fileTabInfo.setAnalystEmailByPosition(index, fileAnalystEmail)
                fileTabInfo.writeMetadata()
                break

        if found == False:
            raise ValueError('Madrigal file %s not found in %s' % (filename, os.path.join(expDir, 'fileTab.txt')))


    def overwriteMadrigalFile(self,
                              expDir,
                              madFilename,
                              notify = True):
        """overwriteMadrigalFile overwrites a file already registered in fileTab.txt.

        Automatically updates expTab.txt with any start or end experiment times.

        Inputs:

            expDir - full path to experiment directory
        
            madFilename - full path to the new Madrigal file.  Basename must match that of one in fileTab.txt.
            
            notify - if True (the default), send a message to all registered users.  If False, do not.
    

        Returns: None

        Affects: Overwrites existing Madrigal file.  May modify expTab.txt with new start/end times.
        Also updated any cached files.
        """
        # verify this file registered
        try:
            fileTabInfo = madrigal.metadata.MadrigalMetaFile(self.__madDB, os.path.join(expDir, 'fileTab.txt'))
        except:
            raise ValueError('Unable to open fileTab.txt in %s' % (expDir))

        # get basename
        filename = os.path.basename(madFilename)

        # search all Madrigal files in fileTab.txt for right file
        found = False
        fileCount = fileTabInfo.getFileCount()

        for index in range(fileCount):
            thisFilename = fileTabInfo.getFilenameByPosition(index)
            if thisFilename == filename:
                found = True
                break

        if found == False:
            raise ValueError('%s not found in %s' % (filename, os.path.join(expDir, 'fileTab.txt')))

        # cp madFilename to new directory
        shutil.copy(madFilename, os.path.join(expDir, filename))
        os.chmod(os.path.join(expDir, filename), 0o664)

        # rm and re-populate overview
        os.remove(os.path.join(expDir, 'overview', filename + '.summary'))
        fileInfo = madrigal.data.MadrigalFile(os.path.join(expDir, filename), self.__madDB)
        
        cachedName = os.path.join(expDir, 'overview', filename + '.txt')
        if os.access(cachedName + '.gz', os.R_OK):
            os.remove(cachedName + '.gz')
            madrigal.cedar.convertToText(os.path.join(expDir, filename), cachedName)
            subprocess.check_call(['gzip', cachedName])
            
        cachedName = os.path.join(expDir, 'overview', filename + '.nc')
        if os.access(cachedName, os.R_OK):
            os.remove(cachedName)
            madrigal.cedar.convertToNetCDF4(os.path.join(expDir, filename), cachedName)

        # update expTab.txt against all registered files
        self.updateExpTab(expDir)
        
        if notify:
            # get expPath without MAD ROOT
            expPath = expDir[expDir.find('experiments'):]
            if expPath[-1] == '/':
                expPath = expPath[:-1]
            madNotify = MadrigalNotify(self.__madDB)
            message = 'You requested to be notified when the experiment: %s from the Madrigal site %s was updated. The file %s has been modified in that experiment at %s.'  \
                % (expPath, self.__madDB.getTopLevelUrl(), os.path.basename(madFilename), str(datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S')))
            message += 'Please contact %s if you wish to be unregistered.' % (self.__madDB.getContactEmail())
            
            userList = self._userData.getRegisteredUsers(expPath)
            if len(userList) > 0:
                for user in userList:
                    try:
                        madNotify.notify(user, message, 'Update to Madrigal experiment you registered interest in')
                    except:
                        print(('Warning - email to %s failed to be sent' % (user)))
                        
            # next deal with users registered for instrument in general
            madExpObj = madrigal.metadata.MadrigalExperiment(self.__madDB, os.path.join(expDir, 'expTab.txt'))
            kinst = madExpObj.getKinstByPosition(0)
            userList = self._userData.getRegisteredInstUsers(kinst)
            if len(userList) > 0:
                for user in userList:
                    try:
                        madNotify.notify(user, message, 'Update to Madrigal instrument data you registered interest in')
                    except:
                        print(('Warning - email to %s failed to be sent' % (user)))



    def removeMadrigalFile(self,
                           expDir,
                           madFilename,
                           allowMissing=False):
        """removeMadrigalFile removes a file already registered in fileTab.txt.

        Automatically updates expTab.txt with any start or end experiment times.

        Inputs:

            expDir - full path to experiment directory
        
            madFilename - Name of Madrigal file to be removed.  Basename must match that of one in fileTab.txt.
            
            allowMissing - if True, remove file successfully even if the file cannot be removed from expDir. If False,
                the default, raise an error if file does not exist.

        Returns: None

        Affects: Removes existing Madrigal file and removes its line from fileTab.txt.  May modify expTab.txt
        with new start/end times
        """
        # verify this file registered
        try:
            fileTabInfo = madrigal.metadata.MadrigalMetaFile(self.__madDB, os.path.join(expDir, 'fileTab.txt'))
        except:
            raise ValueError('Unable to open fileTab.txt in %s' % (expDir))

        # get basename
        filename = os.path.basename(madFilename)

        # delete line from fileTab.txt
        fileTabInfo.deleteRowByFilename(filename)

        # write new version
        fileTabInfo.writeMetadata()
        
        # state variable to detect deletion of missing file
        was_missing = False

        # rm filename and overview data
        try:
            os.remove(os.path.join(expDir, filename))
        except:
            if allowMissing:
                print(('Warning: Unable to remove %s' % (os.path.join(expDir, filename))))
                was_missing = True
            else:
                raise
        try:
            os.remove(os.path.join(expDir, 'overview', filename + '.summary'))
        except:
            pass
        
        # sometimes conversion to Madrigal 3 left a copy in overview - remove this too if its there
        try:
            os.remove(os.path.join(expDir, 'overview', filename))
        except:
            pass
        
        
        cachedName = os.path.join(expDir, 'overview', filename + '.txt')
        if os.access(cachedName + '.gz', os.R_OK):
            os.remove(cachedName + '.gz')
            
        cachedName = os.path.join(expDir, 'overview', filename + '.nc')
        if os.access(cachedName, os.R_OK):
            os.remove(cachedName)

        # update expTab.txt against all registered files
        if not was_missing:
            self.updateExpTab(expDir)
        
        



    def addWebFile(self,
                   expDir,
                   source,
                   relativePath):
        """addWebFile writes a non-Madrigal file meant to be displayed on the web to somewhere within a Madrigal experiment directory.

            All needed directories will be created if needed.

            Inputs:

                expDir - full path to experiment directory

                source - local web file to write to Madrigal

                relativePath - path relative to expDir to write source file to.  If relativePath ends
                with /, then basename from source used.  Otherwise, basename from relativePath used.

            Returns: None

            Affects: writes a non-Madrigal file to expDir on Madrigal
        """
        # verify expDir is a real Madrigal experiment directory
        try:
            fileTabInfo = madrigal.metadata.MadrigalMetaFile(self.__madDB, os.path.join(expDir, 'fileTab.txt'))
        except:
            raise ValueError('%s not a valid experiment directory - no fileTab.txt' % (expDir))

        if not os.access(os.path.join(expDir, os.path.dirname(relativePath)), os.R_OK):
            # make all dirs
            os.umask(0000)
            os.makedirs(os.path.join(expDir, os.path.dirname(relativePath)), 0o777)
            
        if os.path.basename(relativePath) == '':
            shutil.copy(source, os.path.join(expDir, relativePath, os.path.basename(source)))
            os.chmod(os.path.join(expDir, os.path.dirname(relativePath), os.path.basename(source)), 0o664)
        else:
            shutil.copy(source, os.path.join(expDir, relativePath))
            os.chmod(os.path.join(expDir, relativePath), 0o664)

        

    def updateExpTab(self, expDir):
        """updateExpTab rewrites expTab.txt based on all the Madrigal files registered in fileTab.txt.

        Inputs:

            expDir - full path to experiment directory

        Returns: None

        Affects: rewrites expTab.txt in expDir based on all the Madrigal files registered in fileTab.txt.
        """
        try:
            fileTabInfo = madrigal.metadata.MadrigalMetaFile(self.__madDB, os.path.join(expDir, 'fileTab.txt'))
        except:
            raise ValueError('Unable to open fileTab.txt in %s' % (expDir))

        try:
            expInfo = madrigal.metadata.MadrigalExperiment(self.__madDB, os.path.join(expDir, 'expTab.txt'))
        except:
            raise ValueError('Unable to open expTab.txt in %s' % (expDir))

        # we only need to modify start and end times of experiment
        startTime = None
        endTime = None
        # we'd perfer to use default times if possible
        startTimeDefault = None
        endTimeDefault = None

        # search all Madrigal files for earliest time, latest time
        fileCount = fileTabInfo.getFileCount()
        if fileCount == 0:
            return

        for index in range(fileCount):
            thisMadfilename = fileTabInfo.getFilenameByPosition(index)
            thisCategory = fileTabInfo.getCategoryByPosition(index)
            try:
                fileInfo = madrigal.data.MadrigalFile(os.path.join(expDir, thisMadfilename))
            except:
                print('WARNING: Problem with expDir=%s and thisMadfilename=%s' % (expDir, thisMadfilename))
                continue
            thisStartTimeList = fileInfo.getEarliestTime()
            thisStartTime = datetime.datetime(thisStartTimeList[0],
                                              thisStartTimeList[1],
                                              thisStartTimeList[2],
                                              thisStartTimeList[3],
                                              thisStartTimeList[4],
                                              thisStartTimeList[5])
            thisEndTimeList = fileInfo.getLatestTime()
            thisEndTime = datetime.datetime(thisEndTimeList[0],
                                            thisEndTimeList[1],
                                            thisEndTimeList[2],
                                            thisEndTimeList[3],
                                            thisEndTimeList[4],
                                            thisEndTimeList[5])

            if startTime == None:
                startTime = thisStartTime
            else:
                if startTime > thisStartTime:
                    startTime = thisStartTime

            if endTime == None:
                endTime = thisEndTime
            else:
                if endTime < thisEndTime:
                    endTime = thisEndTime
                    
            if thisCategory == 1:
                if startTimeDefault == None:
                    startTimeDefault = thisStartTime
                else:
                    if startTimeDefault > thisStartTime:
                        startTimeDefault = thisStartTime
    
                if endTimeDefault == None:
                    endTimeDefault = thisEndTime
                else:
                    if endTimeDefault < thisEndTime:
                        endTimeDefault = thisEndTime

        # modify expTab.txt and write
        if startTimeDefault:
            expInfo.setExpStartDateTimeByPosition(startTimeDefault)
        else:
            expInfo.setExpStartDateTimeByPosition(startTime)
        if endTimeDefault:
            expInfo.setExpEndDateTimeByPosition(endTimeDefault)
        else:
            expInfo.setExpEndDateTimeByPosition(endTime)
        expInfo.writeMetadata()


    def updateMaster(self, skipGeo=False):
        """updateMaster is a method to update the local metadata.

        Replaces the former tcl script.

        Gathers data from experiment directories into metadata/expTab.txt and metadata/fileTab.txt.
        Also gathers metadata from OpenMadrigal to update metadata/expTabAll.txt and metadata/fileAllTab.txt,
        to update high level metadata siteTab.txt, instTab.txt, instType.txt, madCatTab.txt, parcods.tab.
        Also updates geophysical data.
        """
        binDir = os.path.join(self.__madDB.getMadroot(), 'bin')

        # update geophysical data
        if not skipGeo:
            print('*** Checking for any geophysical file updates ***')
            cmd = os.path.join(binDir, 'checkGeoUpdate.py')
            os.system(cmd)
        else:
            print('Warning - skipping updating geophysical files too often will make them out of date.')
        
        print('*** Updating local metadata ***')
        self.__updateLocalMetadata__()
        print('*** Updating metadata from other Madrigal sites ***')
        self.__updateGlobalMetadata__()
        print('*** Checking OpenMadrigal for any metadata updates ***')
        self.__checkOpenMadrigalMetadata__()

        # instParmTab.txt
        print('*** Rebuilding instParmTab.txt ***')
        obj = madrigal.metadata.MadrigalInstrumentParameters(self.__madDB)
        obj.rebuildInstParmTable()
        
        # instKindatTab.txt
        print('*** Rebuilding instKindatTab.txt ***')
        obj = madrigal.metadata.MadrigalInstrumentKindats(self.__madDB)
        obj.rebuildInstKindatTable()

        print('updateMaster complete...')



    def __updateLocalMetadata__(self):
        """__updateLocalMetadata__ is a private method to update metadata/expTab.txt and metadata/fileTab.txt
        from the local metadata in the experiments[0-9]* directory
        """
        localSiteID = self.__madDB.getSiteID() # used to check that experiments do not have wrong siteID
        
        metaDict = {}
        metaDict['expText'] = [] # a list of text lines of combined expTab.txt file
        metaDict['expIds'] = [] # a list of all expIds found in locals dirs
        metaDict['expIds2'] = [] # a list of all expIds found summary expDir.txt
        metaDict['fileText'] = [] # a list of text lines of combined fileTab.txt file
        metaDict['presentCount'] = 0 # experiment count so far
        metaDict['totalCount'] = 0 # count of all experiments found in first count
        metaDict['localSiteId'] = localSiteID
        
        # make sure metaDict['expIds'] and metaDict['expIds2'] contains minimum value
        metaDict['expIds'].append(self.__madDB.getSiteID() * 10000000)
        metaDict['expIds2'].append(self.__madDB.getSiteID() * 10000000)
        
        # get all experiment directories
        expDirList = self.__madDB.getExperimentDirs()
        
        # first walk to to simply fill out metaDict['expIds']
        for thisExpDir in expDirList:
            if not os.path.isdir(thisExpDir):
                continue
            for root, dirs, files, in os.walk(thisExpDir):
                         self.__walkExpDirIds__(metaDict, root, dirs + files)
        
        for thisExpDir in expDirList:
            if not os.path.isdir(thisExpDir):
                continue
            for root, dirs, files in os.walk(thisExpDir):
                         self.__walkExpDir__(metaDict, root, dirs + files)

        # update expTab.txt
        f = open(os.path.join(self.__madDB.getMadroot(), 'metadata/expTab.txt'), 'w', encoding='utf-8')
        delimiter = ''
        f.write(delimiter.join(metaDict['expText']))
        f.close()
        
        # now sort it by date to speed searches
        madExpObj = madrigal.metadata.MadrigalExperiment(self.__madDB)
        madExpObj.sortByDateSite()
        madExpObj.writeMetadata(os.path.join(self.__madDB.getMadroot(), 'metadata/expTab.txt'))

        # update fileTab.txt
        f = open(os.path.join(self.__madDB.getMadroot(), 'metadata/fileTab.txt'), 'w', encoding='utf-8')
        f.write(delimiter.join(metaDict['fileText']))
        f.close()


    def __updateGlobalMetadata__(self):
        """__updateGlobalMetadata__ is a private method to update metadata/expTabAll.txt and metadata/fileTabAll.txt
        from the main madrigal server.
        """

        expTabAll = ''
        fileTabAll = ''

        localSiteID = self.__madDB.getSiteID()

        siteList = self.__madSite.getSiteList()

        for site in siteList:
            siteID = site[0]
            # skip local site
            if siteID == localSiteID:
                continue
            siteName = site[1]
            siteDir = '%s_%i' % (siteName, siteID)

            expMetadataFile = os.path.join(siteDir, 'expTab.txt')
            fileMetadataFile = os.path.join(siteDir, 'fileTab.txt')

            try:
                thisExpText  = self.__openMad.getMetadataFromOpenMadrigal(expMetadataFile)
                thisFileText = self.__openMad.getMetadataFromOpenMadrigal(fileMetadataFile)
            except:
                continue

            expTabAll += thisExpText
            fileTabAll += thisFileText

        # append local data
        f = open(os.path.join(self.__madDB.getMadroot(), 'metadata/expTab.txt'))
        expTabAll += f.read()
        f.close()

        f = open(os.path.join(self.__madDB.getMadroot(), 'metadata/fileTab.txt'))
        fileTabAll += f.read()
        f.close()

        # write *All.txt files
        f = open(os.path.join(self.__madDB.getMadroot(), 'metadata/expTabAll.txt'), 'w', encoding='utf-8')
        f.write(expTabAll)
        f.close()
        
        # now sort it by date to speed searches
        madExpObj = madrigal.metadata.MadrigalExperiment(self.__madDB,
                                                         os.path.join(self.__madDB.getMadroot(), 
                                                                      'metadata/expTabAll.txt'))
        madExpObj.sortByDateSite()
        madExpObj.writeMetadata(os.path.join(self.__madDB.getMadroot(), 'metadata/expTabAll.txt'))


        f = open(os.path.join(self.__madDB.getMadroot(), 'metadata/fileTabAll.txt'), 'w', encoding='utf-8')
        f.write(fileTabAll)
        f.close()
        
        # rewrite instData.txt
        self._updateInstData()


    def __checkOpenMadrigalMetadata__(self):
        """__checkOpenMadrigalMetadata__ is a method that check the openmadrigal site for any
        updates to the following metadata files:

            1. siteTab.txt - the list of all Madrigal installations
            2. instTab.txt - the list of all Madrigal instruments
            3. instType.txt - the list of all instrument categories

        If an update is available, and the existing metadata file is an old one, it will be updated.
        However, if the local Madrigal adminstrator edits one of these files, then the file will
        not be updated.  If you want to change these files, it is best to contact the OpenMadrigal
        development administrator (madrigal@haystack.mit.edu)
        """
        metadataFiles = ('metadata3/siteTab.txt', 'metadata/instTab.txt', 'metadata/instType.txt')
        url = 'http://cedar.openmadrigal.org/compareToArchive.py?filePath=%s&fileTextMd5=%s'
        
        for metadataFile in metadataFiles:
            localMetadataFile = os.path.join(self.__madDB.getMadroot(), 'metadata', os.path.basename(metadataFile))
            archivePath = 'madroot/%s' % (metadataFile)
            
            f = open(localMetadataFile)
            text = f.read().encode('utf-8')
            f.close()
            textMd5 = hashlib.md5(text)
            md5Str = textMd5.hexdigest() # md5 checksum of local metadata file

            thisUrl = url % (archivePath, md5Str)

            f = urllib.request.urlopen(thisUrl, timeout=200)
            result = f.read().decode('utf-8')
            f.close()

            items = result.split() # first item is latest revision tag, second is matching revision tag
            if len(items) != 2:
                raise IOError('Problem with url %s' % (thisUrl))

            if items[0] != 'None' and items[0] == items[1]:
                # everything is up to date
                continue

            if items[0] == 'None':
                # failed to find this metadata file
                raise IOError('Problem with url %s' % (thisUrl))

            if items[1] == 'None':
                # this metadata file must have been locally editted, print warning
                print('Metadata file %s has been locally edited - contact the OpenMadrigal administrator at madrigal@haystack.mit.edu to update central metadata' % (metadataFile))
                continue

            # this metadata file needs updating
            print('Downloading revised version of metadata file %s from OpenMadrigal' % (metadataFile))
            text = self.__openMad.getLatestSubversionVersion(os.path.join('madroot',metadataFile))
            f = open(localMetadataFile, 'w', encoding='utf-8')
            f.write(text)
            f.close()
            
    def _updateInstData(self):
        """_updateInstData updates the summary files instData.txt and instDataPriv.txt based on 
        latest version of expTabAll.txt
        
        instData.txt and instDataPriv.txt has three comma-delimited columns:
            1. siteId
            2. kinst of non-test experiments at that siteID (only if site has most data)
            3. space separated ordered list of years with data available
            
        instDataPriv.txt includes local private data, in addition to public.
        
        Note that a kinst will only listed at one site.  If there is local data, it will be listed as
        local.  If it listed at more than one non-local site, it will be listed at the one with more
        experiments.
        """
        summDict = {} # local dict with keys=siteID, value=dict with key=kinst, value = years with non-test data
        summDictPriv = {} # same as summDict, but includes local private data
        kinstDict = {} # local dict with keys = kinst, value = dict with key=siteID, value = number of
                       # experiments.  Used to print warning if inst at multiple sites and to decide which
                       # site to incliude
        madExpObj = madrigal.metadata.MadrigalExperiment(self.__madDB,
                                                         os.path.join(self.__madDB.getMadroot(),
                                                                      'metadata/expTabAll.txt'))
        
        archive_sites = set([8,10])
        localSiteID = self.__madDB.getSiteID()
        for i in range(madExpObj.getExpCount()):
            siteID = madExpObj.getExpSiteIdByPosition(i)
            security = madExpObj.getSecurityByPosition(i)
            url = madExpObj.getExpUrlByPosition(i)
            # skip test experiments
            if self.__madDB.isTestExperiment(url, siteID) and security == 0:
                continue
            # skip all non-local archived data
            if siteID != localSiteID and security in (2,3):
                continue
            kinst = madExpObj.getKinstByPosition(i)
            sDTList = madExpObj.getExpStartDateTimeByPosition(i)
            eDTList = madExpObj.getExpEndDateTimeByPosition(i)
            # create a year list
            yearList = list(range(sDTList[0], eDTList[0]+1))
            # add to summDictPriv if not already there
            if siteID not in summDictPriv:
                summDictPriv[siteID] = {}
            if kinst not in summDictPriv[siteID]:
                summDictPriv[siteID][kinst] = [] # empty list of years
            for thisYear in yearList:
                if thisYear not in summDictPriv[siteID][kinst]:
                    summDictPriv[siteID][kinst].append(thisYear)
            # add to summDict if not already there and not private
            if security in (0,2):
                if siteID not in summDict:
                    summDict[siteID] = {}
                if kinst not in summDict[siteID]:
                    summDict[siteID][kinst] = [] # empty list of years
                for thisYear in yearList:
                    if thisYear not in summDict[siteID][kinst]:
                        summDict[siteID][kinst].append(thisYear)
            # add to kinstDict
            if kinst not in kinstDict:
                kinstDict[kinst] = {}
            if siteID not in list(kinstDict[kinst].keys()):
                kinstDict[kinst][siteID] = 1
                kinstSet = set(kinstDict[kinst].keys())
                if len(kinstSet.difference(archive_sites)) > 1:
                    print(('Note: kinst %i found at multiple non-archive sites: %s' % (kinst, str(kinstSet.difference(archive_sites)))))
            else:
                kinstDict[kinst][siteID] += 1
                
        # write to output files
        delimiter = ' '
        outputNames = ('instData.txt', 'instDataPriv.txt')
        dictList = (summDict, summDictPriv)
        for i in range(len(outputNames)):
            f = open(os.path.join(self.__madDB.getMadroot(), 'metadata', outputNames[i]), 'w', encoding='utf-8')
            thisDict = dictList[i]
            siteIDKeys = list(thisDict.keys())
            siteIDKeys.sort()
            for siteID in siteIDKeys:
                kinstKeys = list(thisDict[siteID].keys())
                kinstKeys.sort()
                for kinst in kinstKeys:
                    
                    # verify this kinst/siteID combination is the desired one
                    if localSiteID != siteID: 
                        if localSiteID in list(kinstDict[kinst].keys()):
                            continue # this is not local site, but local site has data and local site always wins
                        if len(kinstDict[kinst]) > 1:
                            accept = True # test whether this non-local site has the most experiments
                            thisCount = kinstDict[kinst][siteID]
                            for thisKey in list(kinstDict[kinst].keys()):
                                if kinstDict[kinst][thisKey] > thisCount:
                                    accept = False
                                    break
                            if not accept:
                                continue
                            
                    # this data is accepted - write it out
                    yearsList = thisDict[siteID][kinst]
                    yearsList.sort()
                    yearsStrList = [str(year) for year in yearsList]
                    yearsStr = delimiter.join(yearsStrList)
                    f.write('%i,%i,%s\n' % (siteID, kinst, yearsStr))
                    
            f.close()
        
                

    def __walkExpDir__(self, arg, dirname, names):
        """__walkExpDir__ is a private method called by os.walk.  arg is a dict with keys:
        1. extText = text of combined expTab.txt to be appended to
        2. fileText = text of combined fileTab.txt to be appended to
        3. presentCount = total experiments done so far
        4. localSiteId = local site id (int)
        5. expIds =  a list of all expIds found in locals dirs - set earlier
        6. expIds2 = a list of all expIds used so far in summary expDir.txt
        
        Sets values in arg
        """
        if 'expTab.txt' not in names:
            return

        # defines allowed experiment directory names
        dirConvStr1 = '/experiments[0-9]*/[0-9][0-9][0-9][0-9]/[a-z][a-z0-9][a-z0-9]/[^/]*'

        # check that dirname follows rule experiments[0-9]*/YYYY/sss/*
        madroot = self.__madDB.getMadroot()
        if madroot[-1] == '/':
            madroot = madroot[:-1]
        startIndex = len(madroot)
        testDir = dirname[startIndex:]
        if re.match(dirConvStr1, testDir) == None:
            return
        
        # make sure we only descend four levels
        count = 0
        items = testDir.split('/')
        for item in items:
            if len(item) > 0:
                count += 1
        if count != 4:
            return
        try:
            
            expObj = madrigal.metadata.MadrigalExperiment(self.__madDB,
                                                          os.path.join(dirname, 'expTab.txt'))
    
            # skip it if security == -1 (ignore flag)
            if expObj.getSecurityByPosition(0) == -1:
                return
    
            # skip if wrong site id
            if expObj.getExpSiteIdByPosition(0) != arg['localSiteId']:
                print('Warning: Experiment %s has wrong site id = %i.  This site id = %i' % \
                      (dirname, expObj.getExpSiteIdByPosition(0), arg['localSiteId']))
                return
    
            # modify experiment id if needed
            thisExpId = expObj.getExpIdByPosition(0)
            isNewExpId = False
            if thisExpId <= self.__madDB.getSiteID() * 10000000 or thisExpId in arg['expIds2']:
                # find a new unique id
                maxId1 = numpy.max(numpy.array(arg['expIds']))
                maxId2 = numpy.max(numpy.array(arg['expIds2']))
                newExpId = int(max([maxId1,maxId2])) + 1
                    
                expObj.setExpIdByPosition(0, newExpId)
                arg['expText'].append(str(expObj))
                # sanity check
                if newExpId in arg['expIds2']:
                    raise ValueError('Duplicate id %i' % (newExpId))
                arg['expIds2'].append(newExpId)
                expObj.writeMetadata()
                isNewExpId = True
                
                print(('Updated metadata in %s' % (str(dirname))))
                
                
            elif thisExpId in arg['expIds2']:
                raise ValueError('found unexpected duplicate expId %i' % (thisExpId))
            
            else:
                # normal case - this id already set and not changing
                arg['expText'].append(str(expObj))
                # sanity check
                if thisExpId in arg['expIds2']:
                    raise ValueError('Duplicate id %i' % (thisExpId))
                arg['expIds2'].append(thisExpId)
            
            arg['presentCount'] += 1
    
            if 'fileTab.txt' not in names:
                print('Info: Experiment %s has no fileTab.txt' % (dirname))
                return
    
            fileObj = madrigal.metadata.MadrigalMetaFile(self.__madDB,
                                                         os.path.join(dirname, 'fileTab.txt'))
    
            # set expId for all files
            for i in range(fileObj.getFileCount()):
                if isNewExpId:
                    fileObj.setExpIdByPosition(i, newExpId)
                else:
                    fileObj.setExpIdByPosition(i, thisExpId)
                
    
            arg['fileText'].append(str(fileObj))
            
            if isNewExpId:
                fileObj.writeMetadata()
                
            if arg['presentCount'] % 1000 == 0:
                print(('Done %i out of %i exps' % (arg['presentCount'], arg['totalCount'])))
            
        except:
            print((' *** Exception encounted in experiment directory %s ***' % (dirname)))
            raise
        
        
    def __walkExpDirIds__(self, arg, dirname, names):
        """__walkExpDirIds__ is a private method called by os.walk.  arg is a dict with keys:
        1. extText = text of combined expTab.txt to be appended to
        2. fileText = text of combined fileTab.txt to be appended to
        3. presentCount = total experiments done so far
        4. localSiteId = local site id (int)
        5. expIds =  a list of all expIds found in locals dirs - to be set here
        6. expIds2 = a list of all expIds used so far in summary expDir.txt
        
        Sets values in arg
        """
        if 'expTab.txt' not in names:
            return

        # defines allowed experiment directory names
        dirConvStr1 = '/experiments[0-9]*/[0-9][0-9][0-9][0-9]/[a-z][a-z0-9][a-z0-9]/[^/]*'

        # check that dirname follows rule experiments[0-9]*/YYYY/sss/*
        madroot = self.__madDB.getMadroot()
        if madroot[-1] == '/':
            madroot = madroot[:-1]
        startIndex = len(madroot)
        testDir = dirname[startIndex:]
        if re.match(dirConvStr1, testDir) == None:
            return
        
        # make sure we only descend four levels
        count = 0
        items = testDir.split('/')
        for item in items:
            if len(item) > 0:
                count += 1
        if count != 4:
            return
        try:
            
            expObj = madrigal.metadata.MadrigalExperiment(self.__madDB,
                                                          os.path.join(dirname, 'expTab.txt'))
    
            # skip it if security == -1 (ignore flag)
            if expObj.getSecurityByPosition(0) == -1:
                return
    
            # skip if wrong site id
            if expObj.getExpSiteIdByPosition(0) != arg['localSiteId']:
                print('Warning: Experiment %s has wrong site id = %i.  This site id = %i' % \
                      (dirname, expObj.getExpSiteIdByPosition(0), arg['localSiteId']))
                return
    
            # add this id if unique
            thisExpId = expObj.getExpIdByPosition(0)
            if int(thisExpId) not in arg['expIds']:
                arg['expIds'].append(int(thisExpId))
                
            # increment totalCount
            arg['totalCount'] += 1
            
        except:
            print((' *** Exception encounted in experiment directory %s ***' % (dirname)))
            raise

Ancestors (in MRO)

Static methods

def __init__(

self, madDB=None)

init initializes MadrigalDBAdmin

Inputs: madDB - Existing MadrigalDB object. Default = None.

Returns: void

Affects:

Sets self.__madDB to MadrigalDB object
Sets self.__madInst to MadrigalInstrument object
def __init__(self, madDB = None):
    """__init__ initializes MadrigalDBAdmin
    
    Inputs: madDB - Existing MadrigalDB object.  Default = None.
    
    Returns: void
    Affects:
        Sets self.__madDB to MadrigalDB object
        Sets self.__madInst to MadrigalInstrument object
    """
    if madDB == None:
        self.__madDB = madrigal.metadata.MadrigalDB()
    else:
        self.__madDB = madDB
    self.__madInst = madrigal.metadata.MadrigalInstrument(self.__madDB)
    self.__openMad = madrigal.openmadrigal.OpenMadrigal(self.__madDB)
    self.__madSite = madrigal.metadata.MadrigalSite(self.__madDB)
    self._userData = madrigal.ui.userData.MadrigalUserData(self.__madDB)

def addMadrigalFile(

self, expDir, madFilename, permission, fileDesc, category=1, kindat=None, notify=True, fileAnalyst='', fileAnalystEmail='', createCachedText=False, createCachedNetCDF4=False, updateToMad3=False, acceptOldSummary=False)

addMadrigalFile adds a new file to an experiment using metadata read from madFilename.

Inputs:

expDir - full path to experiment directory (as returned by createMadriogalExperiment)

madFilename - full path to the complete Madrigal file.  Basename will be maintained.

permission - 0 (public) or 1 (private).

fileDesc - file description

category - 1=default, 2=variant, 3=history, or 4=realtime. Default is 1 (default file)

kindat - if not None (the default), use this kindat instead of what is found in the file.

notify - if True (the default), send a message to all registered users.  If False, do not.

fileAnalyst - full name of file Analyst.  Default is ''

fileAnalystEmail - email of file Analyst.  Default is ''

createCachedText - if True, add cached text file in overview/<basename>.txt.gz.  If False,
    no cached file.

createCachedNetCDF4 - if True, add cached netCDF4 file in overview/<basename>.nc.  If False,
    no cached file.

updateToMad3 - if False (the default), error raised if madFilename non-Hdf5 file. If True, try to
    convert madFilename to Madrigal with .hdf5 extension before loading.

acceptOldSummary - if True, accept an old summary file. Used mainly for upgrading to Madrigal 3. Default
    is False.

Returns: None

def addMadrigalFile(self,
                    expDir,
                    madFilename,
                    permission,
                    fileDesc,
                    category = 1,
                    kindat = None,
                    notify = True,
                    fileAnalyst = '',
                    fileAnalystEmail = '',
                    createCachedText=False, createCachedNetCDF4=False,
                    updateToMad3=False, acceptOldSummary=False):
    """addMadrigalFile adds a new file to an experiment using metadata read from madFilename.
    Inputs:
        expDir - full path to experiment directory (as returned by createMadriogalExperiment)
    
        madFilename - full path to the complete Madrigal file.  Basename will be maintained.
        
        permission - 0 (public) or 1 (private). 
        
        fileDesc - file description
        category - 1=default, 2=variant, 3=history, or 4=realtime. Default is 1 (default file)
        kindat - if not None (the default), use this kindat instead of what is found in the file.
        
        notify - if True (the default), send a message to all registered users.  If False, do not.
        
        fileAnalyst - full name of file Analyst.  Default is ''
        
        fileAnalystEmail - email of file Analyst.  Default is ''
        
        createCachedText - if True, add cached text file in overview/.txt.gz.  If False,
            no cached file.
            
        createCachedNetCDF4 - if True, add cached netCDF4 file in overview/.nc.  If False,
            no cached file.
          
        updateToMad3 - if False (the default), error raised if madFilename non-Hdf5 file. If True, try to
            convert madFilename to Madrigal with .hdf5 extension before loading.
            
        acceptOldSummary - if True, accept an old summary file. Used mainly for upgrading to Madrigal 3. Default
            is False.
    Returns: None
        
    """
    base, ext = os.path.splitext(madFilename)
    if updateToMad3:
        if ext not in ('.hdf5', '.h5', '.hdf5'):
            madFilename = convertMad2FileToMad3(madFilename, self.__madDB, fileDesc)
    elif ext not in ('.hdf5', '.h5', '.hdf5'):
        raise ValueError('called addMadrigalFile with non-Hdf5 file <%s> and updateToMad3 False' \
            % (madFilename))
            
    fileInfo = madrigal.data.MadrigalFile(madFilename, self.__madDB, acceptOldSummary=acceptOldSummary)
    # kindat
    if kindat == None:
        kindatList = fileInfo.getKindatList()
        if len(kindatList) == 0:
            raise ValueError('No kindat values found in file')
        if len(kindatList) > 1:
            raise ValueError('More than one kindat value found in file: %s' % (str(kindatList)))
        kindat = kindatList[0]
    else:
        kindat = int(kindat)
    # permission
    if permission not in (0,1):
        raise ValueError('permission must be either 0 or 1, not %s' % (str(permission)))
    # fileDesc
    if type(fileDesc) not in (bytes, str) and fileDesc != None:
        raise ValueError('fileDesc not a string')
    if fileDesc == None:
        fileDesc = ''
    # check that fileDesc does not illegally contain a comma
    if fileDesc.find(',') != -1:
        raise ValueError('fileDesc string in fileTab.txt cannot contain a comma: <%s> is illegal' % (fileDesc))
    # category
    if category not in (1,2,3,4):
        raise ValueError('category must be 1=default, 2=variant, 3=history, or 4=realtime; not %s' % (str(category)))
    # hasCatalog and hasHeader
    catStr = fileInfo.getCatalogHeaderStr()
    if len(catStr) > 0:
        hasCatalog = 1
        hasHeader = 0
    else:
        hasCatalog = 0
        hasHeader = 0
        
    if fileAnalyst.find(',') != -1:
        raise ValueError('fileAnalyst cannot contain a comma')
    
    if fileAnalystEmail.find(',') != -1:
        raise ValueError('fileAnalystEmail cannot contain a comma')
    
    # all the arguments check out - add line to fileTab.txt
    if not os.access(os.path.join(expDir, 'fileTab.txt'), os.R_OK):
        raise ValueError(' file %s does not yet exist' % (os.path.join(expDir, 'fileTab.txt')))
    # check that this is a new filename
    fileTabObj = madrigal.metadata.MadrigalMetaFile(self.__madDB, os.path.join(expDir, 'fileTab.txt'))
    for i in range(fileTabObj.getFileCount()):
        filename = fileTabObj.getFilenameByPosition(i)
        if filename == os.path.basename(madFilename):
            raise ValueError('File %s already exists - must be deleted first' % (filename))
        
    # fileTab.txt
    nowUT = datetime.datetime.utcnow()
    nowDate = nowUT.strftime('%Y%m%d')
    nowTime = nowUT.strftime('%H%M%S')
    fileTabText = os.path.basename(madFilename)
    fileTabText += ',0,%i,%i,0,%i,%i,%s,%s,' % (kindat,category,hasCatalog,hasHeader,nowDate,nowTime)
    fileTabText += '%s,%i,%s,%s\n' % (fileDesc, permission,
                                      fileAnalyst, fileAnalystEmail)
    # write fileTab.txt
    f = open(os.path.join(expDir, 'fileTab.txt'), 'a')
    f.write(fileTabText)
    f.close()                                                                                                                
    # cp madFilename to new directory
    shutil.copy2(madFilename, os.path.join(expDir, os.path.basename(madFilename)))
    os.chmod(os.path.join(expDir, os.path.basename(madFilename)), 0o664)
    # populate overview
    overviewFile = os.path.join(os.path.dirname(madFilename), 'overview', os.path.basename(madFilename) + '.summary')
    if os.access(overviewFile, os.R_OK):
        # make sure overview exist
        overviewDir = os.path.join(expDir, 'overview')
        if not os.access(overviewDir, os.W_OK):
            os.mkdir(overviewDir)
            os.chmod(overviewDir, 0o777)
        shutil.copy2(overviewFile, overviewDir)
    else:
        fileInfo = madrigal.data.MadrigalFile(os.path.join(expDir, os.path.basename(madFilename)), self.__madDB,
                                              acceptOldSummary=acceptOldSummary)
            
    if createCachedText:
        cachedName = os.path.join(expDir, 'overview', os.path.basename(madFilename) + '.txt')
        madrigal.cedar.convertToText(madFilename, cachedName)
        subprocess.check_call(['gzip', cachedName])
        
    if createCachedNetCDF4:
        cachedName = os.path.join(expDir, 'overview', os.path.basename(madFilename) + '.nc')
        madrigal.cedar.convertToNetCDF4(madFilename, cachedName)
    # update expTab.txt against all registered files
    self.updateExpTab(expDir)
    
    
    if notify:
        # get expPath without MAD ROOT
        expPath = expDir[expDir.find('experiments'):]
        if expPath[-1] == '/':
            expPath = expPath[:-1]
        madNotify = MadrigalNotify(self.__madDB)
        message = 'You requested to be notified when the experiment: %s from the Madrigal site %s was updated. A new file %s has been added to that experiment at %s. '  \
            % (expPath, self.__madDB.getTopLevelUrl(), os.path.basename(madFilename), str(datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S')))
        message += 'Please contact %s if you wish to be unregistered.' % (self.__madDB.getContactEmail())
            
        # first deal with users registered for just this experiment
        userList = self._userData.getRegisteredUsers(expPath)
        if len(userList) > 0:
            for user in userList:
                try:
                    madNotify.notify(user, message, 'Update to Madrigal experiment you registered interest in')
                except:
                    print(('Warning - email to %s failed to be sent' % (user)))
                    
        # next deal with users registered for instrument in general
        madExpObj = madrigal.metadata.MadrigalExperiment(self.__madDB, os.path.join(expDir, 'expTab.txt'))
        kinst = madExpObj.getKinstByPosition(0)
        userList = self._userData.getRegisteredInstUsers(kinst)
        if len(userList) > 0:
            for user in userList:
                try:
                    madNotify.notify(user, message, 'Update to Madrigal instrument data you registered interest in')
                except:
                    print(('Warning - email to %s failed to be sent' % (user)))

def addWebFile(

self, expDir, source, relativePath)

addWebFile writes a non-Madrigal file meant to be displayed on the web to somewhere within a Madrigal experiment directory.

All needed directories will be created if needed.

Inputs:

expDir - full path to experiment directory

source - local web file to write to Madrigal

relativePath - path relative to expDir to write source file to.  If relativePath ends
with /, then basename from source used.  Otherwise, basename from relativePath used.

Returns: None

Affects: writes a non-Madrigal file to expDir on Madrigal

def addWebFile(self,
               expDir,
               source,
               relativePath):
    """addWebFile writes a non-Madrigal file meant to be displayed on the web to somewhere within a Madrigal experiment directory.
        All needed directories will be created if needed.
        Inputs:
            expDir - full path to experiment directory
            source - local web file to write to Madrigal
            relativePath - path relative to expDir to write source file to.  If relativePath ends
            with /, then basename from source used.  Otherwise, basename from relativePath used.
        Returns: None
        Affects: writes a non-Madrigal file to expDir on Madrigal
    """
    # verify expDir is a real Madrigal experiment directory
    try:
        fileTabInfo = madrigal.metadata.MadrigalMetaFile(self.__madDB, os.path.join(expDir, 'fileTab.txt'))
    except:
        raise ValueError('%s not a valid experiment directory - no fileTab.txt' % (expDir))
    if not os.access(os.path.join(expDir, os.path.dirname(relativePath)), os.R_OK):
        # make all dirs
        os.umask(0000)
        os.makedirs(os.path.join(expDir, os.path.dirname(relativePath)), 0o777)
        
    if os.path.basename(relativePath) == '':
        shutil.copy(source, os.path.join(expDir, relativePath, os.path.basename(source)))
        os.chmod(os.path.join(expDir, os.path.dirname(relativePath), os.path.basename(source)), 0o664)
    else:
        shutil.copy(source, os.path.join(expDir, relativePath))
        os.chmod(os.path.join(expDir, relativePath), 0o664)

def appendRTMadrigalFile(

self, expDir, rtFilename, rtFile)

appendRTMadrigalFile used to allow appending to a realtime Madrigal file. No longer possible with Hdf5 format.

Inputs:

expDir - full path to experiment directory (as returned by createRTExperiment)

rtFilename - basename of realtime file to be writtem

rtFile - a string containing the new realtime file contents

Always raises IOError

def appendRTMadrigalFile(self,
                         expDir,
                         rtFilename,
                         rtFile):
    """appendRTMadrigalFile used to allow appending to a realtime Madrigal file. No longer possible with Hdf5 format.
    Inputs:
    
        expDir - full path to experiment directory (as returned by createRTExperiment)
        
        rtFilename - basename of realtime file to be writtem
        
        rtFile - a string containing the new realtime file contents
    Always raises IOError
    """
    raise IOError('The method appendRTMadrigalFile no longer supported by Madrigal 3.0')

def changeExpStatus(

self, expDir, expUrl=None, expName=None, siteID=None, startDatetime=None, endDatetime=None, inst=None, security=None, PI=None, PIEmail=None)

changeExpStatus is used to change attributes in expTab.txt. If None, no change.

Inputs:

expDir - full path to experiment directory. Required.  Example:
   "/opt/madrigal/experiments/1998/mlh/20jan98". If None, do not change.

expUrl - must be in form <cgi base>/madtoc/YYYY/<3 letter lower case inst code>/<expDir>
           example: http://www.haystack.mit.edu/cgi-bin/madtoc/1997/mlh/03dec97g.
           If None, do not change.

expName - experiment name.  Quotes required if contains spaces.  Example: "World Day"
            If None, do not change.

siteID - Madrigal siteID (int) of where data will be stored.  Error raised if not the siteID
           of the local Madrigal site. Example: 4. If None, do not change.

startDatetime - new start datetime of experiment (UT). If None, do not change.

endDatetime - new end datetime of experiment (UT). If None, do not change.

inst - new instrument code (int).  Example: 30. If None, do not change.  Prints
        warning if not found in instTab.txt

security - new security code.  Allowed values are 0 for public, 1 for private (limited IP range access)
        -1 for ignore, 2 for archived experiment, 3 for private (limited IP range access) archived
        experiment. If None, do not change.

PI - name of PI. If None, no change

PIEmail - PI email.  If None, no change
def changeExpStatus(self,
                    expDir,
                    expUrl=None,
                    expName = None,
                    siteID = None,
                    startDatetime = None,
                    endDatetime = None,
                    inst = None,
                    security = None,
                    PI = None,
                    PIEmail = None):
    """changeExpStatus is used to change attributes in expTab.txt.  If None, no change.
    Inputs:
    
        expDir - full path to experiment directory. Required.  Example:
           "/opt/madrigal/experiments/1998/mlh/20jan98". If None, do not change.
        
        expUrl - must be in form /madtoc/YYYY/<3 letter lower case inst code>/
                   example: http://www.haystack.mit.edu/cgi-bin/madtoc/1997/mlh/03dec97g.
                   If None, do not change.
        expName - experiment name.  Quotes required if contains spaces.  Example: "World Day"
                    If None, do not change.
        siteID - Madrigal siteID (int) of where data will be stored.  Error raised if not the siteID
                   of the local Madrigal site. Example: 4. If None, do not change.
        startDatetime - new start datetime of experiment (UT). If None, do not change.
        endDatetime - new end datetime of experiment (UT). If None, do not change.
        inst - new instrument code (int).  Example: 30. If None, do not change.  Prints
                warning if not found in instTab.txt
        security - new security code.  Allowed values are 0 for public, 1 for private (limited IP range access)
                -1 for ignore, 2 for archived experiment, 3 for private (limited IP range access) archived
                experiment. If None, do not change.
                
        PI - name of PI. If None, no change
        
        PIEmail - PI email.  If None, no change
        
    """
    try:
        expTabInfo = madrigal.metadata.MadrigalExperiment(self.__madDB, os.path.join(expDir, 'expTab.txt'))
    except:
        raise ValueError('Unable to open expTab.txt in %s' % (expDir))
    # be sure only one experiment
    if expTabInfo.getExpCount() != 1:
        raise ValueError('expTab.txt in %s has %i experiments, should have exactly 1' % (expDir,
                                                                                          expTabInfo.getExpCount()))
    # expUrl
    if expUrl != None:
        # print warning if not this directory
        index = expUrl.find('/madtoc/')
        thisDir = expUrl[index+8:]
        if thisDir[-1] == '/':
            thisDir = thisDir[:-1]
        if expDir.find(thisDir) == -1:
            raise ValueError('The experiment url you are setting this experiment to <%s> conflicts with experiment directory %s' % (expUrl, expDir))
        expTabInfo.setExpUrlByPosition(0, expUrl)
    # expName
    if expName != None:
        expTabInfo.setExpNameByPosition(0, expName)
    # siteID
    if siteID != None:
        siteID = int(siteID)
        if siteID != self.__madDB.getSiteID():
            raise ValueError('Setting experiment to a siteID %i different from this site\'s id %i' % (siteID,
                                                                                                                self.__madDB.getSiteID()))
        expTabInfo.setExpSiteIdByPosition(0, siteID)
    # startDatetime
    if startDatetime != None:
        # verify before endDatetime if that also being set
        if endDatetime != None:
            if startDatetime > endDatetime:
                raise ValueError('startDatetime %s must be before endDatetime %s' % (str(startDatetime),
                                                                                      str(endDatetime)))
        expTabInfo.setExpStartDateTimeByPosition(startDatetime, 0)
    # endDatetime
    if endDatetime != None:
        expTabInfo.setExpEndDateTimeByPosition(endDatetime, 0)
    # inst
    if inst != None:
        inst = int(inst)
        if self.__madInst.getInstrumentName(inst) == None:
            print('WARNING: instrument %i not found in instTab.txt' % (inst))
        expTabInfo.setExpKinstByPosition(0, inst)
    # security
    if security != None:
        security = int(security)
        if security not in (-1, 0, 1, 2, 3):
            raise ValueError('security must be in (-1, 0, 1, 2, 3), not %i' % (inst))
        expTabInfo.setSecurityByPosition(0, security)
        
    # PI
    if PI != None:
        expTabInfo.setPIByPosition(0, PI)
        
    # PIEmail
    if PIEmail != None:
        expTabInfo.setPIEmailByPosition(0, PIEmail)
    # everything successfully changed - write new values
    expTabInfo.writeMetadata()

def changeFileStatus(

self, expDir, filename, category=None, fileDesc=None, permission=None, kindat=None, fileAnalyst=None, fileAnalystEmail=None)

changeFileStatus is used to change category, fileDesc, or permission of a register file in fileTab.txt.

Inputs:

expDir - full path to experiment directory

filename - basename of existing Madrigal file already registered in fileTab.txt

permission - 0 (public) or 1 (private). If None (default), leave unchanged.

fileDesc - file description. If None (default), leave unchanged.

category - 1=default, 2=variant, or 3=history. If None (default), leave unchanged.

kindat - kindat (int). If None (default), leave unchanged.

fileAnalyst - name of file analyst.  If None (default), leave unchanged.

fileAnalystEmail - email of file analyst.  If None (default), leave unchanged.
def changeFileStatus(self,
                     expDir,
                     filename,
                     category = None,
                     fileDesc = None,
                     permission = None,
                     kindat = None,
                     fileAnalyst = None,
                     fileAnalystEmail = None):
    """changeFileStatus is used to change category, fileDesc, or permission of a register file in fileTab.txt.
    Inputs:
        expDir - full path to experiment directory 
    
        filename - basename of existing Madrigal file already registered in fileTab.txt
        
        permission - 0 (public) or 1 (private). If None (default), leave unchanged. 
        
        fileDesc - file description. If None (default), leave unchanged.
        category - 1=default, 2=variant, or 3=history. If None (default), leave unchanged.
        kindat - kindat (int). If None (default), leave unchanged.
        
        fileAnalyst - name of file analyst.  If None (default), leave unchanged.
        
        fileAnalystEmail - email of file analyst.  If None (default), leave unchanged.
    """
    try:
        fileTabInfo = madrigal.metadata.MadrigalMetaFile(self.__madDB, os.path.join(expDir, 'fileTab.txt'))
    except:
        raise ValueError('Unable to open fileTab.txt in %s' % (expDir))
    # be sure filename is basename
    filename = os.path.basename(filename)
    # search all Madrigal files in fileTab.txt for right file
    found = False
    fileCount = fileTabInfo.getFileCount()
    for index in range(fileCount):
        thisFilename = fileTabInfo.getFilenameByPosition(index)
        if thisFilename == filename:
            found = True
            if category != None:
                fileTabInfo.setCategoryByPosition(index, category)
            if fileDesc != None:
                fileTabInfo.setStatusByPosition(index, fileDesc)
            if permission != None:
                fileTabInfo.setAccessByPosition(index, permission)
            if kindat != None:
                fileTabInfo.setKindatByPosition(index, kindat)
            if fileAnalyst != None:
                fileTabInfo.setAnalystByPosition(index, fileAnalyst)
            if fileAnalystEmail != None:
                fileTabInfo.setAnalystEmailByPosition(index, fileAnalystEmail)
            fileTabInfo.writeMetadata()
            break
    if found == False:
        raise ValueError('Madrigal file %s not found in %s' % (filename, os.path.join(expDir, 'fileTab.txt')))

def createMadrigalExperiment(

self, madFilename, expTitle, permission, fileDesc, instCode=None, category=1, optChar='', dirName=None, kindat=None, experimentsDirNum=None, PI='', PIEmail='', fileAnalyst='', fileAnalystEmail='', createCachedText=False, createCachedNetCDF4=False, notify=True, updateToMad3=False)

createMadrigalExperiment creates a new experiment on Madrigal using metadata read from madFilename.

Inputs:

madFilename - full path to the complete Madrigal file.  Basename will be maintained.

expTitle - experiment title

permission - 0 (public) or 1 (private) or -1 (ignore).

fileDesc - file description

instCode - instrument code.  If default (None), instrument code is taken from file, but error
is thrown if more than one kinst found.

category - 1=default, 2=variant, 3=history, or 4=realtime. Default is 1 (default file)

optChar - optional character to be added to experiment directory if no dirName
          given.  If dirName argument given, this argument ignored.  optChar
          is used if the default directory name DDmmmYY is used for
          more than one experiment created for a given instrument on a given day.
          For example, if --optChar=h for a MLH experiment on September 12, 2005,
          then the experiment directory created would be experiments/2005/mlh/12sep05h.

dirName - directory name to use for experiment.  If None (the default), the directory
          name will be the default name DDmmmYY[optChar].  Cannot contain "/"

kindat - if not None (the default), use this kindat instead of what is found in the file.

experimentsDirNum - the number to be appended to the experiments directory, if experiments
          directory being used is of the form experiments[0-9]* instead of just
          experiments.  For example, if experimentsDirNum is 7, then the experiment
          would be created in MADROOT/experiments7 instead of MADROOT/experiments.

PI- full name of principal investigator.  The default is ''

PIEmail - email of principal investigator.  The default is ''

fileAnalyst -full name of file analyst.  The default is ''

fileAnalystEmail - email of file analyst,.  The default is ''

createCachedText - if True, add cached text file in overview/<basename>.txt.gz.  If False,
    no cached file.

createCachedNetCDF4 - if True, add cached netCDF4 file in overview/<basename>.nc.  If False,
    no cached file.

notify - if True (the default), send a message to all registered users.  If False, do not.

updateToMad3 - if False (the default), error raised if madFilename non-Hdf5 file. If True, try to
    convert madFilename to Madrigal with .hdf5 extension before loading.

Returns:

Full path to directory created
def createMadrigalExperiment(self,
                             madFilename,
                             expTitle,
                             permission,
                             fileDesc,
                             instCode = None,
                             category = 1,
                             optChar = '',
                             dirName = None,
                             kindat = None,
                             experimentsDirNum=None,
                             PI='', PIEmail='', 
                             fileAnalyst='', fileAnalystEmail='',
                             createCachedText=False, createCachedNetCDF4=False,
                             notify = True, updateToMad3=False):
    """createMadrigalExperiment creates a new experiment on Madrigal using metadata read from madFilename.
    Inputs:
    
        madFilename - full path to the complete Madrigal file.  Basename will be maintained.
        
        expTitle - experiment title
        
        permission - 0 (public) or 1 (private) or -1 (ignore). 
        
        fileDesc - file description
        instCode - instrument code.  If default (None), instrument code is taken from file, but error
        is thrown if more than one kinst found.
        category - 1=default, 2=variant, 3=history, or 4=realtime. Default is 1 (default file)
        optChar - optional character to be added to experiment directory if no dirName
                  given.  If dirName argument given, this argument ignored.  optChar
                  is used if the default directory name DDmmmYY is used for
                  more than one experiment created for a given instrument on a given day.
                  For example, if --optChar=h for a MLH experiment on September 12, 2005,
                  then the experiment directory created would be experiments/2005/mlh/12sep05h.
        dirName - directory name to use for experiment.  If None (the default), the directory
                  name will be the default name DDmmmYY[optChar].  Cannot contain "/"
        kindat - if not None (the default), use this kindat instead of what is found in the file.
        
        experimentsDirNum - the number to be appended to the experiments directory, if experiments
                  directory being used is of the form experiments[0-9]* instead of just
                  experiments.  For example, if experimentsDirNum is 7, then the experiment
                  would be created in MADROOT/experiments7 instead of MADROOT/experiments.
                  
        PI- full name of principal investigator.  The default is ''
        
        PIEmail - email of principal investigator.  The default is ''
        
        fileAnalyst -full name of file analyst.  The default is ''
        
        fileAnalystEmail - email of file analyst,.  The default is ''
        
        createCachedText - if True, add cached text file in overview/.txt.gz.  If False,
            no cached file.
            
        createCachedNetCDF4 - if True, add cached netCDF4 file in overview/.nc.  If False,
            no cached file.
            
        notify - if True (the default), send a message to all registered users.  If False, do not.
        
        updateToMad3 - if False (the default), error raised if madFilename non-Hdf5 file. If True, try to
            convert madFilename to Madrigal with .hdf5 extension before loading.
        
    Returns:
        Full path to directory created
        
    """
    # check optChar
    if type(optChar) not in (bytes,str):
        raise ValueError('optChar must be an empty or a one character string, not %s' % (str(optChar)))
    if len(optChar) > 1:
        raise ValueError('optChar must be an empty or a one character string, not %s' % (str(optChar)))
    
    base, ext = os.path.splitext(madFilename)
    if updateToMad3:
        if ext not in ('.hdf5', '.h5', '.hdf5'):
            madFilename = convertMad2FileToMad3(madFilename, self.__madDB, fileDesc)
    elif ext not in ('.hdf5', '.h5', '.hdf5'):
        raise ValueError('called createMadrigalExperiment with non-Hdf5 file <%s> and updateToMad3 False' \
            % (madFilename))
            
    fileInfo = madrigal.data.MadrigalFile(madFilename, self.__madDB)
    # get startTime
    sTime = fileInfo.getEarliestTime()
    startTime = datetime.datetime(sTime[0],sTime[1],sTime[2],sTime[3],sTime[4],sTime[5])
    # get endTime
    eTime = fileInfo.getLatestTime()
    endTime = datetime.datetime(eTime[0],eTime[1],eTime[2],eTime[3],eTime[4],eTime[5])
    
    # get instrument mnemonic and instCode
    if instCode == None:
        kinstList = fileInfo.getKinstList()
        if len(kinstList) == 0:
            raise ValueError('No kinst values found in file')
        if len(kinstList) > 1:
            raise ValueError('More than one kinst value found in file: %s' % (str(kinstList)))
        instCode = kinstList[0]
    instMnemonic = self.__madInst.getInstrumentMnemonic(instCode)
    instDesc = self.__madInst.getInstrumentName(instCode)
    if instMnemonic == None:
        raise ValueError('Unable to find mnemonic for kinst %i' % (instCode))
    # expTitle
    if type(expTitle) not in (bytes, str) and expTitle != None:
        raise ValueError('expTitle not a string')
    if expTitle == None:
        expTitle = ''
    if expTitle.find(',') != -1:
        raise ValueError('expTitle cannot contain a comma')
    # kindat
    if kindat == None:
        kindatList = fileInfo.getKindatList()
        if len(kindatList) == 0:
            raise ValueError('No kindat values found in file')
        if len(kindatList) > 1:
            raise ValueError('More than one kindat value found in file: %s' % (str(kindatList)))
        kindat = kindatList[0]
    else:
        kindat = int(kindat)
    # permission
    if permission not in (0,1, -1):
        raise ValueError('permission must be either 0 or 1 or -1, not %s' % (str(permission)))
    # fileDesc
    if type(fileDesc) not in  (bytes, str) and fileDesc != None:
        raise ValueError('fileDesc not a string')
    if fileDesc == None:
        fileDesc = ''
    if fileDesc.find(',') != -1:
        raise ValueError('fileDesc cannot contain a comma')
    # category
    if category not in (1,2,3,4):
        raise ValueError('category must be 1=default, 2=variant, 3=history, or 4=realtime; not %s' % (str(category)))
    # hasCatalog and hasHeader
    catStr = fileInfo.getCatalogHeaderStr()
    if len(catStr) > 0:
        hasCatalog = 1
        hasHeader = 1
    else:
        hasCatalog = 0
        hasHeader = 0
        
    # experimentDirNum
    if experimentsDirNum != None:
        experimentsDir = 'experiments%i' % (experimentsDirNum)
        # verify this directory exists
        if not os.access(os.path.join(self.__madDB.getMadroot(), experimentsDir), os.R_OK):
            raise ValueError('no such directory %s' % (experimentsDir))
    else:
        experimentsDir = 'experiments'
        
    # PI
    if type(PI) not in (bytes, str) and PI != None:
        raise ValueError('PI not a string')
    if PI == None:
        PI = ''
    if PI.find(',') != -1:
        raise ValueError('PI cannot contain a comma')
        
    # PIEmail
    if type(PIEmail) not in (bytes, str) and PIEmail != None:
        raise ValueError('PIEmail not a string')
    if PIEmail == None:
        PIEmail = ''
    if PIEmail.find(',') != -1:
        raise ValueError('PIEmail cannot contain a comma')
        
    # fileAnalyst
    if type(fileAnalyst) not in (bytes, str) and fileAnalyst != None:
        raise ValueError('fileAnalyst not a string')
    if fileAnalyst == None:
        fileAnalyst = ''
    if fileAnalyst.find(',') != -1:
        raise ValueError('fileAnalyst cannot contain a comma')
        
    # fileAnalystEmail
    if type(fileAnalystEmail) not in (bytes, str) and fileAnalystEmail != None:
        raise ValueError('fileAnalystEmail not a string')
    if fileAnalystEmail == None:
        fileAnalystEmail = ''
    if fileAnalystEmail.find(',') != -1:
        raise ValueError('fileAnalystEmail cannot contain a comma')
    
    # all the arguments check out - create the directory
    # create experiment dir
    expDir2 = os.path.join('%i' % (startTime.year),
                          instMnemonic)
    
    if dirName == None:
        dirName = startTime.strftime('%d%b%y').lower() + optChar
    else:
        # verify dirName is basename
        if dirName.find('/') != -1:
            raise ValueError('dirName must be base directory name, not %s' % (dirName))
        
    expDir2 = os.path.join(expDir2, dirName)
    expDir = os.path.join(self.__madDB.getMadroot(), experimentsDir, expDir2)
    
    expDir3 = os.path.join(experimentsDir, expDir2)
    # if the directory already exists, raise error
    if os.access(expDir, os.R_OK):
        raise IOError('Directory %s already exists' % (expDir))
    os.makedirs(expDir)
    os.chmod(expDir, 0o775)
    # expTab.txt
    expTabText = '0,' + self.__madDB.getTopLevelUrl()
    if expTabText[-1] != '/':
        expTabText += '/madtoc/'
    else:
        expTabText += 'madtoc/'
    expTabText += '%s,%s,%i,%04i%02i%02i,%02i%02i%02i,%04i%02i%02i,%02i%02i%02i,%i,%i,%s,%s\n' %(expDir3,
                                                                                                 expTitle,
                                                                                                 self.__madDB.getSiteID(),
                                                                                                 startTime.year,
                                                                                                 startTime.month,
                                                                                                 startTime.day,
                                                                                                 startTime.hour,
                                                                                                 startTime.minute,
                                                                                                 startTime.second,
                                                                                                 endTime.year,
                                                                                                 endTime.month,
                                                                                                 endTime.day,
                                                                                                 endTime.hour,
                                                                                                 endTime.minute,
                                                                                                 endTime.second,
                                                                                                 instCode,
                                                                                                 permission,
                                                                                                 PI, PIEmail)
    # write expTab.txt
    f = open(os.path.join(expDir, 'expTab.txt'), 'w', encoding='utf-8')
    f.write(expTabText)
    f.close()
    os.chmod(os.path.join(expDir, 'expTab.txt'), 0o664)
    # fileTab.txt
     # set file times to right now UT
    nowUT = datetime.datetime.utcnow()
    nowDate = nowUT.strftime('%Y%m%d')
    nowTime = nowUT.strftime('%H%M%S')
    fileTabText = os.path.basename(madFilename)
    fileTabText += ',0,%i,%i,0,%i,%i,%s,%s,' % (kindat,category,hasCatalog,hasHeader,nowDate,nowTime)
    fileTabText += '%s,%i,%s,%s\n' % (fileDesc, permission,
                                      fileAnalyst, fileAnalystEmail)
    # write fileTab.txt
    f = open(os.path.join(expDir, 'fileTab.txt'), 'w', encoding='utf-8')
    f.write(fileTabText)
    f.close()
    os.chmod(os.path.join(expDir, 'fileTab.txt'), 0o664)
    # create all writeable directory overview
    os.makedirs(os.path.join(expDir, 'overview'))
    os.chmod(os.path.join(expDir, 'overview'), 0o777)
    # cp madFilename to new directory
    shutil.copy2(madFilename, os.path.join(expDir, os.path.basename(madFilename)))
    os.chmod(os.path.join(expDir, os.path.basename(madFilename)), 0o664)
    # populate overview
    overviewFile = os.path.join(os.path.dirname(madFilename), 'overview', os.path.basename(madFilename) + '.summary')
    if os.access(overviewFile, os.R_OK):
        # make sure overview exist
        overviewDir = os.path.join(expDir, 'overview')
        if not os.access(overviewDir, os.W_OK):
            os.mkdir(overviewDir)
            os.chmod(overviewDir, 0o777)
        shutil.copy2(overviewFile, overviewDir)
    else:
        fileInfo = madrigal.data.MadrigalFile(os.path.join(expDir, os.path.basename(madFilename)), self.__madDB,
                                              acceptOldSummary=acceptOldSummary)
    
    if createCachedText:
        cachedName = os.path.join(expDir, 'overview', os.path.basename(madFilename) + '.txt')
        madrigal.cedar.convertToText(madFilename, cachedName)
        subprocess.check_call(['gzip', cachedName])
        
    if createCachedNetCDF4:
        cachedName = os.path.join(expDir, 'overview', os.path.basename(madFilename) + '.nc')
        madrigal.cedar.convertToNetCDF4(madFilename, cachedName)
    
    if notify:
        # get expPath without MAD ROOT
        expPath = expDir[expDir.find('experiments'):]
        if expPath[-1] == '/':
            expPath = expPath[:-1]
        madNotify = MadrigalNotify(self.__madDB)
        message = 'You requested to be notified when the instrument %s from the Madrigal site %s was updated. A new experiment at %s was created on %s.'  \
            % (instDesc, self.__madDB.getTopLevelUrl(), expDir3, str(datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S')))
        message += 'Please contact %s if you wish to be unregistered.' % (self.__madDB.getContactEmail())
            
        # deal with users registered for instrument in general
        userList = self._userData.getRegisteredInstUsers(instCode)
        if len(userList) > 0:
            for user in userList:
                try:
                    madNotify.notify(user, message, 'Update to Madrigal instrument data you registered interest in')
                except:
                    print(('Warning - email to %s failed to be sent' % (user)))
    return(expDir)

def createRTExperiment(

self, startTime, numDays, instrument, expTitle, rtFilenameList, kindatList, permissionList, fileDescList, optChar='', endTime=None, security=0, dirName=None, experimentsDirNum=None, PI='', PIEmail='', fileAnalystList=None, fileAnalystEmailList=None, notify=True)

createRTExperiment creates a new experiment on Madrigal in preparation for realtime data.

Since the experiment is presumably not yet complete, metadata such as the duration of the experiment must be estimated. This metadata will be overwritten when the first batch file is added.

Inputs:

startTime - experiment start time.  If a number, assumed to be seconds since 1/1/1970.  May also
be a datetime.datetime object

numDays - number of days the experiment is estimated to run.  Ignored if optional endTime given.

instrument - instrument code or 3-letter Madrigal mnenonic

expTitle - experiment title

rtFilenameList - list of realtime filenames to be created. Must all be Hdf5 files.

kindatList - list of ints of kindats for each realtime file.  Len = len(rtFilenameList)

permissionList - list of 0 (public) or 1 (private). Len = len(rtFilenameList)

fileDescList - list of realtime file descriptions

optChar - optional character to be added to experiment directory if no dirName
          given.  If dirName argument given, this argument ignored.  optChar
          is used if the default directory name DDmmmYY is used for
          more than one experiment created for a given instrument on a given day.
          For example, if --optChar=h for a MLH experiment on September 12, 2005,
          then the experiment directory created would be experiments/2005/mlh/12sep05h.

endTime - optional end date and time of experiment.  If a number, assumed to be seconds since
1/1/1970.  May also be a datetime.datetime object

security - experiment security setting.  If 0 (the default) public.  If 1, private.
           If -1, entire experiment ignored.  Any given file permission is the more
           restricted of experiment permission and file permission.

dirName - directory name to use for experiment.  If None (the default), the directory
          name will be the default name DDmmmYY[optChar].  Cannot contain "/"

experimentsDirNum - the number to be appended to the experiments directory, if experiments
          directory being used is of the form experiments[0-9]* instead of just
          experiments.  For example, if experimentsDirNum is 7, then the experiment
          would be created in MADROOT/experiments7 instead of MADROOT/experiments.

PI- full name of principal investigator.  The default is ''

PIEmail - email of principal investigator.  The default is ''

fileAnalystList - list of full names of file analysts, one for each file.  If None, the default, 
    File Analyst = ''

fileAnalystEmailList - list of emails of file analysts, one for each file.  If None, the default, 
    File Analyst email = ''

notify - if True (the default), send a message to all registered users.  If False, do not.

Returns:

Full path to directory created
def createRTExperiment(self,
                       startTime,
                       numDays,
                       instrument,
                       expTitle,
                       rtFilenameList,
                       kindatList,
                       permissionList,
                       fileDescList,
                       optChar = '',
                       endTime = None,
                       security = 0,
                       dirName = None,
                       experimentsDirNum=None,
                       PI='', PIEmail='', 
                       fileAnalystList=None, fileAnalystEmailList=None,
                       notify = True):
    """createRTExperiment creates a new experiment on Madrigal in preparation for realtime data.
    Since the experiment is presumably not yet complete, metadata such as the duration of the experiment
    must be estimated.  This metadata will be overwritten when the first batch file is added.
    Inputs:
    
        startTime - experiment start time.  If a number, assumed to be seconds since 1/1/1970.  May also
        be a datetime.datetime object
        
        numDays - number of days the experiment is estimated to run.  Ignored if optional endTime given.
        
        instrument - instrument code or 3-letter Madrigal mnenonic
        
        expTitle - experiment title
        
        rtFilenameList - list of realtime filenames to be created. Must all be Hdf5 files.
        
        kindatList - list of ints of kindats for each realtime file.  Len = len(rtFilenameList)
        
        permissionList - list of 0 (public) or 1 (private). Len = len(rtFilenameList)
        
        fileDescList - list of realtime file descriptions
        optChar - optional character to be added to experiment directory if no dirName
                  given.  If dirName argument given, this argument ignored.  optChar
                  is used if the default directory name DDmmmYY is used for
                  more than one experiment created for a given instrument on a given day.
                  For example, if --optChar=h for a MLH experiment on September 12, 2005,
                  then the experiment directory created would be experiments/2005/mlh/12sep05h.
        endTime - optional end date and time of experiment.  If a number, assumed to be seconds since
        1/1/1970.  May also be a datetime.datetime object
        security - experiment security setting.  If 0 (the default) public.  If 1, private.
                   If -1, entire experiment ignored.  Any given file permission is the more
                   restricted of experiment permission and file permission.
        dirName - directory name to use for experiment.  If None (the default), the directory
                  name will be the default name DDmmmYY[optChar].  Cannot contain "/"
                  
        experimentsDirNum - the number to be appended to the experiments directory, if experiments
                  directory being used is of the form experiments[0-9]* instead of just
                  experiments.  For example, if experimentsDirNum is 7, then the experiment
                  would be created in MADROOT/experiments7 instead of MADROOT/experiments.
                  
        PI- full name of principal investigator.  The default is ''
        
        PIEmail - email of principal investigator.  The default is ''
        
        fileAnalystList - list of full names of file analysts, one for each file.  If None, the default, 
            File Analyst = ''
        
        fileAnalystEmailList - list of emails of file analysts, one for each file.  If None, the default, 
            File Analyst email = ''
            
        notify - if True (the default), send a message to all registered users.  If False, do not.
    Returns:
        Full path to directory created
        
    """
    # check optChar
    if type(optChar) not in (bytes, str):
        raise ValueError('optChar must be an empty or a one character string, not %s' % (str(optChar)))
    if len(optChar) > 1:
        raise ValueError('optChar must be an empty or a one character string, not %s' % (str(optChar)))
    security = int(security)
    if security not in (-1,0,1):
        raise ValueError('security must be -1, 0, or 1, not %i' % (security))
    
    # convert startTime to datetime if needed
    if type(startTime) in (int, int, float):
        startTime = datetime.datetime.utcfromtimestamp(startTime)
    # create endTime based on numDays
    if endTime != None:
        if type(endTime) in (int, int, float):
            endTime = datetime.datetime.utcfromtimestamp(startTime)
    else:
        if numDays >= 0:
            endTime = startTime + datetime.timedelta(numDays) - datetime.timedelta(0,1)
        else:
            raise ValueError('numDays must not be negative')
    if startTime >= endTime:
        raise ValueError('Experiment start time %s after end time %s' % (str(startTime),
                                                                           str(endTime)))
    
    # get instrument mnemonic and instCode
    try:
        instCode = int(instrument)
        instMnemonic = self.__madInst.getInstrumentMnemonic(instCode)
    except ValueError:
        if len(instrument) != 3:
            raise ValueError('%s not a legal instrument mnemonic' % (str(instrument)))
        instMnemonic = instrument.lower()
        # verify its a legal mnemonic
        instList = self.__madInst.getInstrumentList()
        found = False
        for inst in instList:
            if instMnemonic == inst[1]:
                found = True
                instCode = inst[2]
                break
        if found == False:
            raise ValueError('%s not a legal instrument mnemonic or code' % (str(instrument)))
    if instMnemonic == None:
        raise ValueError('%s not a legal instrument mnemonic or code' % (str(instrument)))
    
    instDesc = self.__madInst.getInstrumentName(instCode)
    # expTitle
    if type(expTitle) not in (bytes, str) and expTitle != None:
        raise ValueError('expTitle not a string')
    if expTitle == None:
        expTitle = ''
    if expTitle.find(',') != -1:
        raise ValueError('expTitle cannot contain a comma')
        
    # PI
    if type(PI) not in (bytes, str) and PI != None:
        raise ValueError('PI not a string')
    if PI == None:
        PI = ''
    if PI.find(',') != -1:
        raise ValueError('PI cannot contain a comma')
        
    # PIEmail
    if type(PIEmail) not in (bytes, str) and PIEmail != None:
        raise ValueError('PIEmail not a string')
    if PIEmail == None:
        PIEmail = ''
    if PIEmail.find(',') != -1:
        raise ValueError('PIEmail cannot contain a comma')
    # rtFilenameList
    if type(rtFilenameList) not in (list, tuple):
        raise ValueError('rtFilenameList not a list or tuple')
    # make sure each rtFilename is a string without /
    for filename in rtFilenameList:
        if type(filename) not in (bytes, str):
            raise ValueError('rtFilenameList must contain strings')
        if filename.find('/') != -1:
            raise ValueError('rtFilenameList must contain strings without /')
        if filename.find(',') != -1:
            raise ValueError('filename cannot contain a comma')
        base, ext = os.path.splitext(filename)
        if ext not in ('.hdf5', '.h5', '.hdf5'):
            raise ValueError('All input files must have a valid Hdf5 extension, <%s> does not' % (filename))
    # kindatList
    if len(kindatList) != len(rtFilenameList):
        raise ValueError('length of kindatList not equal length of rtFilenameList')
    for item in kindatList:
        try:
            int(item)
        except:
            raise ValueError('kindatList must contain integers')
    # permissionList
    if len(permissionList) != len(rtFilenameList):
        raise ValueError('length of permissionList not equal length of rtFilenameList')
    for item in permissionList:
        try:
            permission = int(item)
        except:
            raise ValueError('permissionList must contain integers')
        if permission not in (0,1):
            raise ValueError('permissionList must contain integers of value 0 (public) or 1 (private)')
    
    # fileDescList
    if len(fileDescList) != len(rtFilenameList):
        raise ValueError('length of fileDescList not equal length of rtFilenameList')
    for item in fileDescList:
        if type(item) not in (bytes, str):
            raise ValueError('fileDescList must only contain strings')
        if item.find(',') != -1:
            raise ValueError('fileDesc cannot contain a comma')
        
    # fileAnalystList
    if fileAnalystList:
        if len(fileAnalystList) != len(rtFilenameList):
            raise ValueError('length of fileAnalystList not equal length of rtFilenameList')
        for item in fileAnalystList:
            if type(item) not in (bytes, str):
                raise ValueError('fileAnalystList must only contain strings')
            if item.find(',') != -1:
                raise ValueError('fileAnalyst cannot contain a comma')
            
    # fileAnalystEmailList
    if fileAnalystEmailList:
        if len(fileAnalystEmailList) != len(rtFilenameList):
            raise ValueError('length of fileAnalystEmailList not equal length of rtFilenameList')
        for item in fileAnalystEmailList:
            if type(item) not in (bytes, str):
                raise ValueError('fileAnalystEmailList must only contain strings')
            if item.find(',') != -1:
                raise ValueError('fileAnalystEmail cannot contain a comma')
        
    # experimentDirNum
    if experimentsDirNum != None:
        experimentsDir = 'experiments%i' % (experimentsDirNum)
        # verify this directory exists
        if not os.access(os.path.join(self.__madDB.getMadroot(), experimentsDir), os.R_OK):
            raise ValueError('no such directory %s' % (experimentsDir))
    else:
        experimentsDir = 'experiments'
    # all the arguments check out - create the directory
    # create experiment dir
    expDir2 = os.path.join('%i' % (startTime.year),
                           instMnemonic)
    
    if dirName == None:
        dirName = startTime.strftime('%d%b%y').lower() + optChar
    else:
        # verify dirName is basename
        if dirName.find('/') != -1:
            raise ValueError('dirName must be base directory name, not %s' % (dirName))
    expDir2 = os.path.join(expDir2, dirName)
    expDir = os.path.join(self.__madDB.getMadroot(), experimentsDir, expDir2)
    
    expDir3 = os.path.join(experimentsDir, expDir2)
    # if the directory already exists, raise error
    if os.access(expDir, os.R_OK):
        raise IOError('Directory %s already exists' % (expDir))
    os.makedirs(expDir)
    os.chmod(expDir, 0o775)
    # expTab.txt
    expTabText = '0,' + self.__madDB.getTopLevelUrl()
    if expTabText[-1] != '/':
        expTabText += '/madtoc/'
    else:
        expTabText += 'madtoc/'
    expTabText += '%s,%s,%i,%04i%02i%02i,%02i%02i%02i,%04i%02i%02i,%02i%02i%02i,%i,%i,%s,%s\n' %(expDir3,
                                                                                                 expTitle,
                                                                                                 self.__madDB.getSiteID(),
                                                                                                 startTime.year,
                                                                                                 startTime.month,
                                                                                                 startTime.day,
                                                                                                 startTime.hour,
                                                                                                 startTime.minute,
                                                                                                 startTime.second,
                                                                                                 endTime.year,
                                                                                                 endTime.month,
                                                                                                 endTime.day,
                                                                                                 endTime.hour,
                                                                                                 endTime.minute,
                                                                                                 endTime.second,
                                                                                                 instCode,
                                                                                                 security,
                                                                                                 PI,
                                                                                                 PIEmail)
    # write expTab.txt
    f = open(os.path.join(expDir, 'expTab.txt'), 'w', encoding='utf-8')
    f.write(expTabText)
    f.close()
    os.chmod(os.path.join(expDir, 'expTab.txt'), 0o664)
    # fileTab.txt
    # set file times to right now UT
    nowUT = datetime.datetime.utcnow()
    nowDate = nowUT.strftime('%Y%m%d')
    nowTime = nowUT.strftime('%H%M%S')
    fileTabText = ''
    for index in range(len(rtFilenameList)):
        
        if fileAnalystList and fileAnalystEmailList:
            fileAnalyst = fileAnalystList[index]
            fileAnalystEmail = fileAnalystEmailList[index]
        else:
            fileAnalyst = ''
            fileAnalystEmail = ''
        
        fileTabText += rtFilenameList[index]
        fileTabText += ',0,%i,4,0,0,0,%s,%s,' % (kindatList[index], nowDate, nowTime)
        fileTabText += '%s,%i,%s,%s\n' % (fileDescList[index], permissionList[index],
                                          fileAnalyst, fileAnalystEmail)
    # write fileTab.txt
    f = open(os.path.join(expDir, 'fileTab.txt'), 'w')
    f.write(fileTabText)
    f.close()
    os.chmod(os.path.join(expDir, 'fileTab.txt'), 0o664)
    # create all writeable directory overview
    os.makedirs(os.path.join(expDir, 'overview'))
    os.chmod(os.path.join(expDir, 'overview'), 0o777)
    
    if notify:
        # get expPath without MAD ROOT
        expPath = expDir[expDir.find('experiments'):]
        if expPath[-1] == '/':
            expPath = expPath[:-1]
        madNotify = MadrigalNotify(self.__madDB)
        message = 'You requested to be notified when new data from instrument %s was available. A new realtime experiment for that instrument has been created at %s at %s. '   \
            % (instDesc, self.__madDB.getTopLevelUrl(), str(datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S')))
        message += 'Please contact %s if you wish to be unregistered.' %  (self.__madDB.getContactEmail())
            
        # deal with users registered for instrument in general
        userList = self._userData.getRegisteredInstUsers(instCode)
        if len(userList) > 0:
            for user in userList:
                try:
                    madNotify.notify(user, message, 'Update to Madrigal instrument you registered interest in')
                except:
                    print(('Warning - email to %s failed to be sent' % (user)))
    return(expDir)

def overwriteMadrigalFile(

self, expDir, madFilename, notify=True)

overwriteMadrigalFile overwrites a file already registered in fileTab.txt.

Automatically updates expTab.txt with any start or end experiment times.

Inputs:

expDir - full path to experiment directory

madFilename - full path to the new Madrigal file.  Basename must match that of one in fileTab.txt.

notify - if True (the default), send a message to all registered users.  If False, do not.

Returns: None

Affects: Overwrites existing Madrigal file. May modify expTab.txt with new start/end times. Also updated any cached files.

def overwriteMadrigalFile(self,
                          expDir,
                          madFilename,
                          notify = True):
    """overwriteMadrigalFile overwrites a file already registered in fileTab.txt.
    Automatically updates expTab.txt with any start or end experiment times.
    Inputs:
        expDir - full path to experiment directory
    
        madFilename - full path to the new Madrigal file.  Basename must match that of one in fileTab.txt.
        
        notify - if True (the default), send a message to all registered users.  If False, do not.

    Returns: None
    Affects: Overwrites existing Madrigal file.  May modify expTab.txt with new start/end times.
    Also updated any cached files.
    """
    # verify this file registered
    try:
        fileTabInfo = madrigal.metadata.MadrigalMetaFile(self.__madDB, os.path.join(expDir, 'fileTab.txt'))
    except:
        raise ValueError('Unable to open fileTab.txt in %s' % (expDir))
    # get basename
    filename = os.path.basename(madFilename)
    # search all Madrigal files in fileTab.txt for right file
    found = False
    fileCount = fileTabInfo.getFileCount()
    for index in range(fileCount):
        thisFilename = fileTabInfo.getFilenameByPosition(index)
        if thisFilename == filename:
            found = True
            break
    if found == False:
        raise ValueError('%s not found in %s' % (filename, os.path.join(expDir, 'fileTab.txt')))
    # cp madFilename to new directory
    shutil.copy(madFilename, os.path.join(expDir, filename))
    os.chmod(os.path.join(expDir, filename), 0o664)
    # rm and re-populate overview
    os.remove(os.path.join(expDir, 'overview', filename + '.summary'))
    fileInfo = madrigal.data.MadrigalFile(os.path.join(expDir, filename), self.__madDB)
    
    cachedName = os.path.join(expDir, 'overview', filename + '.txt')
    if os.access(cachedName + '.gz', os.R_OK):
        os.remove(cachedName + '.gz')
        madrigal.cedar.convertToText(os.path.join(expDir, filename), cachedName)
        subprocess.check_call(['gzip', cachedName])
        
    cachedName = os.path.join(expDir, 'overview', filename + '.nc')
    if os.access(cachedName, os.R_OK):
        os.remove(cachedName)
        madrigal.cedar.convertToNetCDF4(os.path.join(expDir, filename), cachedName)
    # update expTab.txt against all registered files
    self.updateExpTab(expDir)
    
    if notify:
        # get expPath without MAD ROOT
        expPath = expDir[expDir.find('experiments'):]
        if expPath[-1] == '/':
            expPath = expPath[:-1]
        madNotify = MadrigalNotify(self.__madDB)
        message = 'You requested to be notified when the experiment: %s from the Madrigal site %s was updated. The file %s has been modified in that experiment at %s.'  \
            % (expPath, self.__madDB.getTopLevelUrl(), os.path.basename(madFilename), str(datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S')))
        message += 'Please contact %s if you wish to be unregistered.' % (self.__madDB.getContactEmail())
        
        userList = self._userData.getRegisteredUsers(expPath)
        if len(userList) > 0:
            for user in userList:
                try:
                    madNotify.notify(user, message, 'Update to Madrigal experiment you registered interest in')
                except:
                    print(('Warning - email to %s failed to be sent' % (user)))
                    
        # next deal with users registered for instrument in general
        madExpObj = madrigal.metadata.MadrigalExperiment(self.__madDB, os.path.join(expDir, 'expTab.txt'))
        kinst = madExpObj.getKinstByPosition(0)
        userList = self._userData.getRegisteredInstUsers(kinst)
        if len(userList) > 0:
            for user in userList:
                try:
                    madNotify.notify(user, message, 'Update to Madrigal instrument data you registered interest in')
                except:
                    print(('Warning - email to %s failed to be sent' % (user)))

def removeMadrigalFile(

self, expDir, madFilename, allowMissing=False)

removeMadrigalFile removes a file already registered in fileTab.txt.

Automatically updates expTab.txt with any start or end experiment times.

Inputs:

expDir - full path to experiment directory

madFilename - Name of Madrigal file to be removed.  Basename must match that of one in fileTab.txt.

allowMissing - if True, remove file successfully even if the file cannot be removed from expDir. If False,
    the default, raise an error if file does not exist.

Returns: None

Affects: Removes existing Madrigal file and removes its line from fileTab.txt. May modify expTab.txt with new start/end times

def removeMadrigalFile(self,
                       expDir,
                       madFilename,
                       allowMissing=False):
    """removeMadrigalFile removes a file already registered in fileTab.txt.
    Automatically updates expTab.txt with any start or end experiment times.
    Inputs:
        expDir - full path to experiment directory
    
        madFilename - Name of Madrigal file to be removed.  Basename must match that of one in fileTab.txt.
        
        allowMissing - if True, remove file successfully even if the file cannot be removed from expDir. If False,
            the default, raise an error if file does not exist.
    Returns: None
    Affects: Removes existing Madrigal file and removes its line from fileTab.txt.  May modify expTab.txt
    with new start/end times
    """
    # verify this file registered
    try:
        fileTabInfo = madrigal.metadata.MadrigalMetaFile(self.__madDB, os.path.join(expDir, 'fileTab.txt'))
    except:
        raise ValueError('Unable to open fileTab.txt in %s' % (expDir))
    # get basename
    filename = os.path.basename(madFilename)
    # delete line from fileTab.txt
    fileTabInfo.deleteRowByFilename(filename)
    # write new version
    fileTabInfo.writeMetadata()
    
    # state variable to detect deletion of missing file
    was_missing = False
    # rm filename and overview data
    try:
        os.remove(os.path.join(expDir, filename))
    except:
        if allowMissing:
            print(('Warning: Unable to remove %s' % (os.path.join(expDir, filename))))
            was_missing = True
        else:
            raise
    try:
        os.remove(os.path.join(expDir, 'overview', filename + '.summary'))
    except:
        pass
    
    # sometimes conversion to Madrigal 3 left a copy in overview - remove this too if its there
    try:
        os.remove(os.path.join(expDir, 'overview', filename))
    except:
        pass
    
    
    cachedName = os.path.join(expDir, 'overview', filename + '.txt')
    if os.access(cachedName + '.gz', os.R_OK):
        os.remove(cachedName + '.gz')
        
    cachedName = os.path.join(expDir, 'overview', filename + '.nc')
    if os.access(cachedName, os.R_OK):
        os.remove(cachedName)
    # update expTab.txt against all registered files
    if not was_missing:
        self.updateExpTab(expDir)

def updateExpTab(

self, expDir)

updateExpTab rewrites expTab.txt based on all the Madrigal files registered in fileTab.txt.

Inputs:

expDir - full path to experiment directory

Returns: None

Affects: rewrites expTab.txt in expDir based on all the Madrigal files registered in fileTab.txt.

def updateExpTab(self, expDir):
    """updateExpTab rewrites expTab.txt based on all the Madrigal files registered in fileTab.txt.
    Inputs:
        expDir - full path to experiment directory
    Returns: None
    Affects: rewrites expTab.txt in expDir based on all the Madrigal files registered in fileTab.txt.
    """
    try:
        fileTabInfo = madrigal.metadata.MadrigalMetaFile(self.__madDB, os.path.join(expDir, 'fileTab.txt'))
    except:
        raise ValueError('Unable to open fileTab.txt in %s' % (expDir))
    try:
        expInfo = madrigal.metadata.MadrigalExperiment(self.__madDB, os.path.join(expDir, 'expTab.txt'))
    except:
        raise ValueError('Unable to open expTab.txt in %s' % (expDir))
    # we only need to modify start and end times of experiment
    startTime = None
    endTime = None
    # we'd perfer to use default times if possible
    startTimeDefault = None
    endTimeDefault = None
    # search all Madrigal files for earliest time, latest time
    fileCount = fileTabInfo.getFileCount()
    if fileCount == 0:
        return
    for index in range(fileCount):
        thisMadfilename = fileTabInfo.getFilenameByPosition(index)
        thisCategory = fileTabInfo.getCategoryByPosition(index)
        try:
            fileInfo = madrigal.data.MadrigalFile(os.path.join(expDir, thisMadfilename))
        except:
            print('WARNING: Problem with expDir=%s and thisMadfilename=%s' % (expDir, thisMadfilename))
            continue
        thisStartTimeList = fileInfo.getEarliestTime()
        thisStartTime = datetime.datetime(thisStartTimeList[0],
                                          thisStartTimeList[1],
                                          thisStartTimeList[2],
                                          thisStartTimeList[3],
                                          thisStartTimeList[4],
                                          thisStartTimeList[5])
        thisEndTimeList = fileInfo.getLatestTime()
        thisEndTime = datetime.datetime(thisEndTimeList[0],
                                        thisEndTimeList[1],
                                        thisEndTimeList[2],
                                        thisEndTimeList[3],
                                        thisEndTimeList[4],
                                        thisEndTimeList[5])
        if startTime == None:
            startTime = thisStartTime
        else:
            if startTime > thisStartTime:
                startTime = thisStartTime
        if endTime == None:
            endTime = thisEndTime
        else:
            if endTime < thisEndTime:
                endTime = thisEndTime
                
        if thisCategory == 1:
            if startTimeDefault == None:
                startTimeDefault = thisStartTime
            else:
                if startTimeDefault > thisStartTime:
                    startTimeDefault = thisStartTime

            if endTimeDefault == None:
                endTimeDefault = thisEndTime
            else:
                if endTimeDefault < thisEndTime:
                    endTimeDefault = thisEndTime
    # modify expTab.txt and write
    if startTimeDefault:
        expInfo.setExpStartDateTimeByPosition(startTimeDefault)
    else:
        expInfo.setExpStartDateTimeByPosition(startTime)
    if endTimeDefault:
        expInfo.setExpEndDateTimeByPosition(endTimeDefault)
    else:
        expInfo.setExpEndDateTimeByPosition(endTime)
    expInfo.writeMetadata()

def updateMaster(

self, skipGeo=False)

updateMaster is a method to update the local metadata.

Replaces the former tcl script.

Gathers data from experiment directories into metadata/expTab.txt and metadata/fileTab.txt. Also gathers metadata from OpenMadrigal to update metadata/expTabAll.txt and metadata/fileAllTab.txt, to update high level metadata siteTab.txt, instTab.txt, instType.txt, madCatTab.txt, parcods.tab. Also updates geophysical data.

def updateMaster(self, skipGeo=False):
    """updateMaster is a method to update the local metadata.
    Replaces the former tcl script.
    Gathers data from experiment directories into metadata/expTab.txt and metadata/fileTab.txt.
    Also gathers metadata from OpenMadrigal to update metadata/expTabAll.txt and metadata/fileAllTab.txt,
    to update high level metadata siteTab.txt, instTab.txt, instType.txt, madCatTab.txt, parcods.tab.
    Also updates geophysical data.
    """
    binDir = os.path.join(self.__madDB.getMadroot(), 'bin')
    # update geophysical data
    if not skipGeo:
        print('*** Checking for any geophysical file updates ***')
        cmd = os.path.join(binDir, 'checkGeoUpdate.py')
        os.system(cmd)
    else:
        print('Warning - skipping updating geophysical files too often will make them out of date.')
    
    print('*** Updating local metadata ***')
    self.__updateLocalMetadata__()
    print('*** Updating metadata from other Madrigal sites ***')
    self.__updateGlobalMetadata__()
    print('*** Checking OpenMadrigal for any metadata updates ***')
    self.__checkOpenMadrigalMetadata__()
    # instParmTab.txt
    print('*** Rebuilding instParmTab.txt ***')
    obj = madrigal.metadata.MadrigalInstrumentParameters(self.__madDB)
    obj.rebuildInstParmTable()
    
    # instKindatTab.txt
    print('*** Rebuilding instKindatTab.txt ***')
    obj = madrigal.metadata.MadrigalInstrumentKindats(self.__madDB)
    obj.rebuildInstKindatTable()
    print('updateMaster complete...')

def writeRTMadrigalFile(

self, expDir, rtFilename, rtFile)

writeRTMadrigalFile writes a realtime Madrigal file to a Madrigal experiment directory.

Fails if rtFilename does not match one listed in fileTab.txt.

Inputs:

expDir - full path to experiment directory (as returned by createRTExperiment)

rtFilename - basename of realtime file to be writtem

rtFile - a string containing the realtime file contents

Returns: None

Raises exception if rtFilename does not match one listed in fileTab.txt.

def writeRTMadrigalFile(self,
                        expDir,
                        rtFilename,
                        rtFile):
    """writeRTMadrigalFile writes a realtime Madrigal file to a Madrigal experiment directory.
    Fails if rtFilename does not match one listed in fileTab.txt.
    Inputs:
    
        expDir - full path to experiment directory (as returned by createRTExperiment)
        
        rtFilename - basename of realtime file to be writtem
        
        rtFile - a string containing the realtime file contents
    Returns: None
    Raises exception if rtFilename does not match one listed in fileTab.txt.
    """
    # verify rtFilename listed in fileTab.txt
    try:
        fileInfo = madrigal.metadata.MadrigalMetaFile(self.__madDB, os.path.join(expDir, 'fileTab.txt'))
    except:
        raise ValueError('Unable to open fileTab.txt in %s' % (expDir))
    
    if fileInfo.getHasCatalogByFilename(rtFilename) == None:
        raise ValueError('Filename %s not found in fileTab.txt' % (rtFilename))
    # okay - write it
    f = open(os.path.join(expDir, rtFilename), 'w', encoding='utf-8')
    f.write(rtFile)
    f.close()
    os.chmod(os.path.join(expDir, rtFilename), 0o664)

class MadrigalError

MadrigalError is an exception class that is thrown for all known errors in using Madrigal Py lib.

Usage example:

import sys, traceback
import madrigal.admin

try:

    test = open('ImportantFile.txt', 'r')

except:

    raise madrigal.admin.MadrigalError('ImportantFile.txt not opened!',
                                        traceback.format_exception(sys.exc_info()[0],
                                                                sys.exc_info()[1],
                                                                sys.exc_info()[2]))
class MadrigalError(Exception):
    """MadrigalError is an exception class that is thrown for all known errors in using Madrigal Py lib.

    Usage example:

        import sys, traceback
        import madrigal.admin
    
        try:
        
            test = open('ImportantFile.txt', 'r')
            
        except:
        
            raise madrigal.admin.MadrigalError('ImportantFile.txt not opened!',
                                                traceback.format_exception(sys.exc_info()[0],
                                                                        sys.exc_info()[1],
                                                                        sys.exc_info()[2]))
    """


    def __init__(self, strInterpretation, exceptionList):
        """ __init__ gathers the interpretation string along with all information from sys.exc_info().

        Inputs: strIntepretation - A string representing the programmer's interpretation of
        why the exception occurred

                exceptionList - a list of strings completely describing the exception.
                Generated by traceback.format_exception(sys.exc_info()[0],
                                                        sys.exc_info()[1],
                                                        sys.exc_info()[2])
        
        Returns: Void.

        Affects: Initializes class member variables _strInterp, _strExcList.

        Exceptions: None.
        """
        
        self._strInterp = strInterpretation
        self._strExcList = exceptionList

        
    def getExceptionStr(self):
        """ getExceptionStr returns a formatted string ready for printing completely describing the exception.

        Inputs: None
        
        Returns: A formatted string ready for printing completely describing the exception.

        Affects: None

        Exceptions: None.
        """
        excStr = 'The following Madrigal Python exception has occurred:\n'
        excStr = excStr + self._strInterp + '\n\n'

        if self._strExcList != None:
            for item in self._strExcList:
                excStr = excStr + str(item) + '\n'

        return excStr
    
    def __str__(self):
        return(self.getExceptionStr())


    def getExceptionHtml(self):
        """ getExceptionHtml returns an Html formatted string completely describing the exception.

        Inputs: None
        
        Returns: A formatted string ready for printing completely describing the exception.

        Affects: None

        Exceptions: None.
        """
        
        excStr = '
The following Madrigal Python exception has occurred:\n
' excStr = excStr + self._strInterp + '\n
\n' if self._strExcList != None: for item in self._strExcList: excStr = excStr + str(item) + '\n
' return excStr

Ancestors (in MRO)

  • MadrigalError
  • builtins.Exception
  • builtins.BaseException
  • builtins.object

Class variables

var args

Static methods

def __init__(

self, strInterpretation, exceptionList)

init gathers the interpretation string along with all information from sys.exc_info().

Inputs: strIntepretation - A string representing the programmer's interpretation of why the exception occurred

    exceptionList - a list of strings completely describing the exception.
    Generated by traceback.format_exception(sys.exc_info()[0],
                                            sys.exc_info()[1],
                                            sys.exc_info()[2])

Returns: Void.

Affects: Initializes class member variables _strInterp, _strExcList.

Exceptions: None.

def __init__(self, strInterpretation, exceptionList):
    """ __init__ gathers the interpretation string along with all information from sys.exc_info().
    Inputs: strIntepretation - A string representing the programmer's interpretation of
    why the exception occurred
            exceptionList - a list of strings completely describing the exception.
            Generated by traceback.format_exception(sys.exc_info()[0],
                                                    sys.exc_info()[1],
                                                    sys.exc_info()[2])
    
    Returns: Void.
    Affects: Initializes class member variables _strInterp, _strExcList.
    Exceptions: None.
    """
    
    self._strInterp = strInterpretation
    self._strExcList = exceptionList

def getExceptionHtml(

self)

getExceptionHtml returns an Html formatted string completely describing the exception.

Inputs: None

Returns: A formatted string ready for printing completely describing the exception.

Affects: None

Exceptions: None.

def getExceptionHtml(self):
    """ getExceptionHtml returns an Html formatted string completely describing the exception.
    Inputs: None
    
    Returns: A formatted string ready for printing completely describing the exception.
    Affects: None
    Exceptions: None.
    """
    
    excStr = '
The following Madrigal Python exception has occurred:\n
' excStr = excStr + self._strInterp + '\n
\n' if self._strExcList != None: for item in self._strExcList: excStr = excStr + str(item) + '\n
' return excStr

def getExceptionStr(

self)

getExceptionStr returns a formatted string ready for printing completely describing the exception.

Inputs: None

Returns: A formatted string ready for printing completely describing the exception.

Affects: None

Exceptions: None.

def getExceptionStr(self):
    """ getExceptionStr returns a formatted string ready for printing completely describing the exception.
    Inputs: None
    
    Returns: A formatted string ready for printing completely describing the exception.
    Affects: None
    Exceptions: None.
    """
    excStr = 'The following Madrigal Python exception has occurred:\n'
    excStr = excStr + self._strInterp + '\n\n'
    if self._strExcList != None:
        for item in self._strExcList:
            excStr = excStr + str(item) + '\n'
    return excStr

class MadrigalNotify

MadrigalNotify is an object used to send messages to an administrator about a Madrigal database.

This object provides functions needed to send messages to an administrator about a Madrigal database, for now only sendAlert, which sends an email to the site administrator found is siteTab.txt (or if not possible, the admin in madrigal.cfg, and finally if all else fails, to root).

Usage example:

import madrigal.admin

try:

    adminObj =  madrigal.admin.MadrigalNotify()
    adminObj.sendAlert('This is important!', 'Important Message')

except madrigal.admin.MadrigalError, e:

    print e.getExceptionStr()

Non-standard Python modules used: None

Exceptions thrown: None - Note that MadrigalNotify tries every trick it knows to avoid throwing exceptions, since this is the class that will generally be called when there is a problem.

Change history:

Written by "Bill Rideout":mailto:wrideout@haystack.mit.edu Dec. 4, 2001

class MadrigalNotify:
    """MadrigalNotify is an object used to send messages to an administrator about a Madrigal database.

    This object provides functions needed to send messages to an administrator about a Madrigal database, for now
    only sendAlert, which sends an email to the site administrator found is siteTab.txt (or if not
    possible, the admin in madrigal.cfg, and finally if all else fails, to root).

    Usage example:

        import madrigal.admin
    
        try:
        
            adminObj =  madrigal.admin.MadrigalNotify()
            adminObj.sendAlert('This is important!', 'Important Message')
            
        except madrigal.admin.MadrigalError, e:
        
            print e.getExceptionStr()


    Non-standard Python modules used:
    None

    Exceptions thrown: None - Note that MadrigalNotify tries every trick it knows to avoid
    throwing exceptions, since this is the class that will generally be called when there is a problem.

    Change history:

    Written by "Bill Rideout":mailto:wrideout@haystack.mit.edu  Dec. 4, 2001
    """
    

    #constants
    __defaultUser  = "root"
    """ Sets the default user to email to when all else fails. """

    __defaultServer  = "localhost"
    """ Sets the default server to send mail when all else fails. """


    def __init__(self, madDB = None):
        """__init__ initializes MadrigalNotify by getting some basic information from MadrigalDB and MadrigalSite.

        Note that MadrigalNotify tries every trick it knows to avoid throwing exceptions, since
        this is the class that will generally be called when there is a problem.

        Inputs: Existing MadrigalDB object, by default = None.
        
        Returns: void

        Affects: Initializes self.__binDir.

        Exceptions: None.
        """

        # get metadata dir
        if madDB == None:
            try:
                thisMadDB = madrigal.metadata.MadrigalDB()
            except:
                # note that the main configuration file is unavailable 
                # the best that can be done is send an email to root using localhost mailserver
                self.__emailAddress = self.__defaultUser
                self.__emailServer  = self.__defaultServer
                thisMadDB = None
        else:
            thisMadDB = madDB
        self.madDB = thisMadDB

        if thisMadDB != None:
            self.__emailServer  = thisMadDB.getMailserver()
            # now try to get email from site metadata, if failure, use config contact info
            try:
                thisSite = madrigal.metadata.MadrigalSite()
                self.__emailAddress = thisSite.getSiteEmail(thisMadDB.getSiteID())
                if self.__emailAddress == None:
                    # couldn't read metadata - use madrigal.cfg
                    self.__emailAddress = thisMadDB.getContactEmail()
            except:
                # couldn't read metadata - use madrigal.cfg
                self.__emailAddress = thisMadDB.getContactEmail()

            #make sure madrigal.cfg worked - if not use root
            if self.__emailAddress == None:
                self.__emailAddress = self.__defaultUser


    def sendAlert(self, message, subject = None):
        """sendAlert sends an email with the given message and optional title.

        Inputs: message (string), and optional title (string)
        
        Returns: void

        Affects: none

        Exceptions: None.
        """

        # set up message
        message = 'From: (Python Madrigal API) ' + self.__emailAddress + '\n' + \
        'To: ' + self.__emailAddress + '\n' + \
        'Subject: ' + str(subject) + '\n' + \
        'Content-type: text/html\n\n' + message

        server = smtplib.SMTP(self.__emailServer)
        server.sendmail(self.__emailAddress.split(',')[0],
                        self.__emailAddress.split(','), message)
        server.quit()
        
        
    def notify(self, email, message, subject):
        """notify sends an email with the given message and title to email.

        Inputs: email (string), message (string), and subject (string)
        
        Returns: void

        Affects: none

        Exceptions: None.
        """

        # set up message
        message = 'From: (Madrigal at %s) ' % (self.madDB.getMadServer()) + self.__emailAddress + '\n' + \
        'To: ' + email + '\n' + \
        'Subject: ' + str(subject) + '\n' + \
        'Content-type: text/html\n\n' + message

        server = smtplib.SMTP(self.__emailServer)
        server.sendmail(self.__emailAddress.split(',')[0],
                        email.split(','), message)
        server.quit()

Ancestors (in MRO)

Static methods

def __init__(

self, madDB=None)

init initializes MadrigalNotify by getting some basic information from MadrigalDB and MadrigalSite.

Note that MadrigalNotify tries every trick it knows to avoid throwing exceptions, since this is the class that will generally be called when there is a problem.

Inputs: Existing MadrigalDB object, by default = None.

Returns: void

Affects: Initializes self.__binDir.

Exceptions: None.

def __init__(self, madDB = None):
    """__init__ initializes MadrigalNotify by getting some basic information from MadrigalDB and MadrigalSite.
    Note that MadrigalNotify tries every trick it knows to avoid throwing exceptions, since
    this is the class that will generally be called when there is a problem.
    Inputs: Existing MadrigalDB object, by default = None.
    
    Returns: void
    Affects: Initializes self.__binDir.
    Exceptions: None.
    """
    # get metadata dir
    if madDB == None:
        try:
            thisMadDB = madrigal.metadata.MadrigalDB()
        except:
            # note that the main configuration file is unavailable 
            # the best that can be done is send an email to root using localhost mailserver
            self.__emailAddress = self.__defaultUser
            self.__emailServer  = self.__defaultServer
            thisMadDB = None
    else:
        thisMadDB = madDB
    self.madDB = thisMadDB
    if thisMadDB != None:
        self.__emailServer  = thisMadDB.getMailserver()
        # now try to get email from site metadata, if failure, use config contact info
        try:
            thisSite = madrigal.metadata.MadrigalSite()
            self.__emailAddress = thisSite.getSiteEmail(thisMadDB.getSiteID())
            if self.__emailAddress == None:
                # couldn't read metadata - use madrigal.cfg
                self.__emailAddress = thisMadDB.getContactEmail()
        except:
            # couldn't read metadata - use madrigal.cfg
            self.__emailAddress = thisMadDB.getContactEmail()
        #make sure madrigal.cfg worked - if not use root
        if self.__emailAddress == None:
            self.__emailAddress = self.__defaultUser

def notify(

self, email, message, subject)

notify sends an email with the given message and title to email.

Inputs: email (string), message (string), and subject (string)

Returns: void

Affects: none

Exceptions: None.

def notify(self, email, message, subject):
    """notify sends an email with the given message and title to email.
    Inputs: email (string), message (string), and subject (string)
    
    Returns: void
    Affects: none
    Exceptions: None.
    """
    # set up message
    message = 'From: (Madrigal at %s) ' % (self.madDB.getMadServer()) + self.__emailAddress + '\n' + \
    'To: ' + email + '\n' + \
    'Subject: ' + str(subject) + '\n' + \
    'Content-type: text/html\n\n' + message
    server = smtplib.SMTP(self.__emailServer)
    server.sendmail(self.__emailAddress.split(',')[0],
                    email.split(','), message)
    server.quit()

def sendAlert(

self, message, subject=None)

sendAlert sends an email with the given message and optional title.

Inputs: message (string), and optional title (string)

Returns: void

Affects: none

Exceptions: None.

def sendAlert(self, message, subject = None):
    """sendAlert sends an email with the given message and optional title.
    Inputs: message (string), and optional title (string)
    
    Returns: void
    Affects: none
    Exceptions: None.
    """
    # set up message
    message = 'From: (Python Madrigal API) ' + self.__emailAddress + '\n' + \
    'To: ' + self.__emailAddress + '\n' + \
    'Subject: ' + str(subject) + '\n' + \
    'Content-type: text/html\n\n' + message
    server = smtplib.SMTP(self.__emailServer)
    server.sendmail(self.__emailAddress.split(',')[0],
                    self.__emailAddress.split(','), message)
    server.quit()

Instance variables

var madDB

Previous: Internal Madrigal Python API   Up: Internal Madrigal Python API   Next: madrigal.cedar