Sleds/buildvm/testtools/filewatcher.py

#!python3
'''
Created on Sep 21, 20-16
author: tpweis
file name: filewatcher.py
This function watches a directory (input argument 1)
when a new file is detected, it is pushed to the Server using WinSCP. 
The method is to create a small "put" script, and execute it.
When the put is complete, the file is moved (locally) to the processed
directory.
'''

import os, time
import datetime
import subprocess
import shutil
import subprocess
import sys
import json
import logging

def helpmessage():
    "function helpmessage"
    print("filewatcher.py <feed directory> <processed directory>")
    logging.warning('Filewatcher initiation or run-time error, msg sent to console')
    logging.warning('Halt')
    sys.exit(1)

def bld_transfer_code(fromdir, todir, filename):
    "function bld_transfer_code"
    # filename argument is the name of the file we are uploading.  The generated 
    # transfer file code file name is formed from this.
    tmp_name = fromdir + "/transfer_" + os.path.basename(filename) + ".script"
    gen_file_name = os.path.normpath(tmp_name)
    logging.info("transfer code location: " + gen_file_name)

    line1 = 'open sftp://root:ijfijfijf%2B%2B@10.133.2.24/ -hostkey="ssh-ed25519 256 82:f3:2b:fb:ce:aa:1a:38:97:05:f1:54:f3:38:eb:b3"'
    line2 = "lcd " + fromdir
    line3 = "cd " + todir
    line4 = 'put "' + filename + '"'
    line5 = "exit"

    # Save the transfer code and return its name
    f = open(gen_file_name,'w')
    print(line1, line2, line3, line4, line5, file= f, sep="\n")
    f.close()

    return gen_file_name

def main():
    "function main"
    logfile = os.path.normpath("/sl/bin/test.log")
    logging.basicConfig(format='%(asctime)s:  %(module)s:  %(levelname)s: %(message)s',
            filename=logfile, level=logging.DEBUG)


    # WinSCP gets the server host name from the transfer code.  This specifices the directory
    # on that machine
    UPLOADdir = "/sllocal/IngestFolder"
    LOGdir = "/sllocal/IngestLogs"
    arglist = sys.argv[1:]
    if(len(arglist)< 1):
        print("Missing Watch Directory argument.  Try again")
        helpmessage()
    else:
        WATCHdir = arglist[0]

    if(len(arglist) < 2):
        print("Missing Processed directory argument.  Try again")
        helpmessage()
    else:
        PROCESSdir =arglist[1]

    # Now ensure that both directorys exist.  And, that the the process directory is different
    # from the watch directory
    if not os.path.isdir(WATCHdir):
        print("First arg (watch directory) " + WATCHdir + " is not a valid directory")
        helpmessage()
    if not os.path.isdir(PROCESSdir):
        print("Second arg (processed directory) " + PROCESSdir + " is not a valid directory")
        helpmessage()

    if WATCHdir == PROCESSdir:
        print("Processed files directory must be different than the Watch directory: " + WATCHdir + " - " + PROCESSdir)
        helpmessage()

    logging.info("Uploading files from " + WATCHdir + ", then moving them to " + PROCESSdir)

    before = dict ([(f, None) for f in os.listdir (WATCHdir)])
    winscp = os.path.normpath("C:\Program Files (x86)\WinSCP\WinSCP.exe")

    if not os.path.isfile(winscp):
        print("cannot locate WinSCP at " + winscp)
        helpmessage()

    # Go into a timed loop to check for new files arriving. 
    interval = 5
    while 1:
        time.sleep (interval)
        logging.debug("Next check for changes in " + str(interval) + " seconds")
        after = dict ([(f, None) for f in os.listdir (WATCHdir)])
        added = [f for f in after if not f in before]
        removed = [f for f in before if not f in after]
        if added: 
            for filename in added:
                logging.debug("Added: " + filename)
                # Find the pieces of the file name.  Because we did an os.listdir,
                # there wont be a directory string in front of the filename.
                # To be certain, use os.path
                filebase = os.path.basename(filename)
                filehead, fileext = os.path.splitext(filebase)

                # Move added file to "processed" directory
                oldfile = WATCHdir + "/" + filename
                newfile = PROCESSdir + "/" + filename 
                logging.info("Move " + oldfile + " to " + newfile)
                try:
                    shutil.move(oldfile,newfile)
                except:
                    time.sleep(3)
                    shutil.move(oldfile,newfile)

                # Build the specific scripting instructions to upload the file
                scriptname = bld_transfer_code(PROCESSdir,UPLOADdir,filebase)
                scriptfile = "/script=" + os.path.normpath(scriptname)
                logfile = "/log=" + os.path.normpath(PROCESSdir + "\\" + filebase + ".log")
                CMD = winscp + " " + logfile + " /ini=nul" + " " + scriptfile
                logging.debug(CMD)
                # Upload added file
                p1 = subprocess.Popen([winscp, logfile, "/ini=nul", scriptfile], stdout=subprocess.PIPE)
                p1.wait()

                # Upload associated log file
                # First, build the specific scripting instructions to upload the log file
                # Notice that the log file is uploaded to a Logs directory, not the input processing folder
                # This is so that SLEDS is not confused with extraneous files.
                scriptname = bld_transfer_code(PROCESSdir, LOGdir, filename + ".log")
                scriptfile = "/script=" + os.path.normpath(scriptname)
                CMD = winscp + " " + logfile + " /ini=nul" + " " + scriptfile
                logging.debug(CMD)
                p3 = subprocess.Popen([winscp, logfile, "/ini=nul", scriptfile], stdout=subprocess.PIPE)
                

        if removed:
            for filename in removed:
                logging.debug ("Removed: " + filename)
        before = after

    sys.exit(0)

    

if __name__ == '__main__':
    # execute only if run as a script
    main()