bobbycar/logdata_visualization/logfix.py

import numpy as np
from datetime import datetime
import time
import argparse
import os.path

parser = argparse.ArgumentParser(description='Copys, renames and fixes logfiles written by bobbycar sd logger.')
parser.add_argument('-i', '--input', type=argparse.FileType('r'), nargs='+', required=True, help="list of input log files")
parser.add_argument('-o', '--output', nargs='?', type=argparse.FileType('w'), help="output filename")
parser.add_argument('-c','--consecutive', action="store_true", help="add consecutive files to input. If the input file ends with a number the following logfiles will be added.")
args = parser.parse_args()



ok=True

def getTimestamp(plines):
    timestampline=-1
    timestampfound=False
    while not timestampfound:
        timestampline+=1
        timestampfound = (plines[timestampline].find('TIMESTAMP:')!=-1)
            
    timestamp=int(plines[timestampline].split('TIMESTAMP:')[1]) #timestamp when file was created

    if (timestampline==-1):
        print("Error: Timestamp not found!")
        exit()

    return timestamp


def filterLines(plines,plinesStarttime=None):

    plines = [x.rstrip("\n") for x in plines] #remove \n
    pcommentlinesMask = [True if x.startswith('#') else False for x in plines] #generate mask for lines with comments

    


    plines=np.array(plines)
    pcommentlinesMask=np.array(pcommentlinesMask)

    if (plinesStarttime is not None): #if starttimelist given, match with pdatalinesOK
        plinesStarttime = plinesStarttime[pcommentlinesMask==False] #get lines with data

    pdatalines = plines[pcommentlinesMask==False] #get lines with data

    

    pheader = pdatalines[0] #header is the first non comment line

    pheaderSize = len(pheader.split(',')) #how many elements are expected per line
    pdatalinesSize = [len(x.split(',')) for x in pdatalines] #count arraysize for every dataline

    if (plinesStarttime is not None): #if starttimelist given, match with pdatalinesOK
        plinesStarttime=plinesStarttime[np.array(pdatalinesSize)==pheaderSize]

    pdatalinesOK = pdatalines[np.array(pdatalinesSize)==pheaderSize]

    if (plinesStarttime is not None): #if starttimelist given, match with pdatalinesOK
        plinesStarttime = [plinesStarttime[i] for i,x in enumerate(pdatalinesOK) if x != pheader]

    pdatalinesOK = [x for x in pdatalinesOK if x != pheader] #exclude header from data lines

    
    
    
    pdatalinesFail = pdatalines[np.array(pdatalinesSize)!=pheaderSize]
    

    plinesSize = [len(x.split(',')) for x in plines] #count arraysize for every dataline
    plinesOK = np.array(plinesSize)==pheaderSize #mask for okay lines (valid for data lines)

    return plines,pheader,pcommentlinesMask,pdatalines,pdatalinesFail,pdatalinesOK,pheaderSize,plinesOK,plinesStarttime


inputFilenames=[]

if (args.consecutive):
    if(len(args.input)!=1):
        parser.error("in consequtive mode exactly one input file is required")
        exit()
    
    nextFilename=args.input[0].name
    while os.path.isfile(nextFilename):
        print(nextFilename+" exists")
        inputFilenames.append(nextFilename)

        digitStartpos=len(nextFilename)-1
        digitEndpos=len(nextFilename)


        
        while (not nextFilename[digitStartpos:digitEndpos].isdigit() and digitStartpos>0 and digitEndpos>0): 
            digitStartpos-=1
            digitEndpos-=1

        while (nextFilename[digitStartpos:digitEndpos].isdigit() and digitStartpos>0 and digitEndpos>0): 
            digitStartpos-=1

        digitStartpos+=1

        
        number=int(nextFilename[digitStartpos:digitEndpos])+1
        nextFilename=nextFilename[0:digitStartpos]+str(number).zfill(digitEndpos-digitStartpos)+nextFilename[digitEndpos:]
        
        

    

else:
    inputFilenames=[x.name for x in args.input]





outputFilename=None
if args.output is not None:
    outputFilename=args.output.name




lines=[]
linesStarttime=[] #offset for every line with timestamp. will be combined to new column
header=""
for inputFilename in inputFilenames:
    print("Reading "+str(inputFilename))
    inputlines=[]
    with open(inputFilename, 'r') as reader:
        inputlines = reader.readlines()

    lines+=inputlines
    
    #Check Headers
    _lines,_header,_,_,_,_,_,_,_=filterLines(inputlines)
    
    if (header==""): #is first header
        header=_header

    assert header==_header, "Header is different!"

    _timestamp=getTimestamp(_lines)
    print("Timestamp="+str(_timestamp))
    _linesStarttime=[_timestamp for x in inputlines] #create as many entries with start timestamp as there are lines in the current file

    linesStarttime+=_linesStarttime
    
    print("Lines in file="+str(len(inputlines)))

assert len(lines)==len(linesStarttime), "Length of lines and linesStarttime does not match"

linesStarttime=np.array(linesStarttime)
lines,header,commentlinesMask,datalines,datalinesFail,datalinesOK,headerSize,linesOK,linesStarttime=filterLines(lines,linesStarttime)

print("Found "+str(len(lines))+" lines")
print(str(np.sum(commentlinesMask))+" comments")
print(str(len(datalinesFail))+" Datalines Failed")
print(str(len(datalinesOK))+" Datalines OK")
print("Header Size is "+str(headerSize))

timestamp=getTimestamp(lines)
filetime = time.strftime('%Y%m%d_%H%M%S', time.localtime(timestamp))

if outputFilename is None:
    outputFilename = filetime+".csv"

#is_dst(datetime(2019, 4, 1), timezone="US/Pacific")
print("Timestamp:"+str(timestamp)+" -> "+str(filetime))
print("UTC: "+ datetime.utcfromtimestamp(timestamp).strftime('%A, %Y-%m-%d %H:%M:%S'))
print("Local Time:"+time.strftime('%A, %Y-%m-%d %H:%M:%S', time.localtime(timestamp)))


print("Writing to: "+str(outputFilename))

print("Size lines="+str(len(lines)))
print("Size commentlinesMask="+str(len(commentlinesMask)))
print("Size datalines="+str(len(datalines)))
print("Size linesOK="+str(len(linesOK)))

header="timestamp,"+header #add timestamp column

writelines = [str(linesStarttime[i]+float(x.split(',')[0]))+","+x for i,x in enumerate(datalinesOK)] #add file timestamp to line time and add column to data

linesWritten = 0
if ok:
    with open(outputFilename, 'w') as writer:
        writer.write(header+"\n") #write header
        for i,line in enumerate(writelines):
            writer.write(line+"\n")
            linesWritten+=1

    print(str(linesWritten)+" lines written to "+str(outputFilename))
else:
    print("Failed!")