add transmission protocol for sd card over serial

controller_teensy/include/display.h

@@ -31,7 +31,6 @@ void display_debugDisplay(ESCSerialComm& escFront, ESCSerialComm& escRear);
 bool display_init(){
   if(!display.begin(SSD1306_SWITCHCAPVCC, SCREEN_ADDRESS)) {
     Serial.println(F("SSD1306 allocation failed"));
-    writeLogComment((unsigned long)millis(), "SSD1306 allocation failed");
     return false;
   }
   display.clearDisplay();

controller_teensy/include/logging.h

@@ -11,6 +11,8 @@ boolean sdcard_available=false;
 boolean datalogging=true;
 String datalogging_filename="UNKNOWN.txt";
 
+uint16_t chunksize=128; //for bulk data transmission
+
 #define LOGGINGINTERVAL 100
 
 bool serialCommandEcho_Enabled=true;
@@ -20,9 +22,11 @@ void loggingLoop(unsigned long loopmillis,ESCSerialComm& escFront, ESCSerialComm
 
 
 void writeLogComment(unsigned long time, String msg);
-void printFileListing();
+void printFileListing(bool pretty);
-void printDirectory(File dir, int numTabs,String parent);
+void printDirectory(File dir, int numTabs,String parent,bool printSize,bool printFullDirectoryNames);
 void printFile(String filename);
+void printFilesize(String filename);
+void transferFile(String filename);
 void removeFile(String filename);
 void serialCommandLoop(unsigned long loopmillis, ESCSerialComm& escFront, ESCSerialComm& escRear);
 float getBatteryVoltage(ESCSerialComm& escFront, ESCSerialComm& escRear);
@@ -318,13 +322,29 @@ void serialCommandLoop(unsigned long loopmillis,ESCSerialComm& escFront, ESCSeri
       if(smessage.equals("test")) {
         Serial.println("OK");
       }else if(smessage.equals("ls")) {
-        printFileListing();
+        printFileListing(false);
+      }else if(smessage.equals("ls -a")) {
+        printFileListing(true);
       }else if(smessage.startsWith("cat")) {
         String _filename=smessage.substring(4);
         if (_filename.length()<3){ //if no filename given, use current log file
           _filename=getLogFilename();
         }
         printFile(_filename);
+      }else if(smessage.startsWith("get")) {
+        String _filename=smessage.substring(4);
+        transferFile(_filename);
+      }else if(smessage.startsWith("sizeof")) {
+        String _filename=smessage.substring(7);
+        printFilesize(_filename);
+      }else if(smessage.startsWith("chunksize")) { //change chunksize for get command
+        String _chunksizestring=smessage.substring(10);
+        long _chunksize=_chunksizestring.toInt();        
+        if (_chunksize>0 && _chunksize<LONG_MAX) { //value ok
+          chunksize=_chunksize;
+        }else{
+          Serial.println("Chunksize value out of range");
+        }
       }else if(smessage.startsWith("rm")) {
         removeFile(smessage.substring(3));
       }else if(smessage.equals("log off")) {
@@ -367,16 +387,20 @@ void serialCommandLoop(unsigned long loopmillis,ESCSerialComm& escFront, ESCSeri
   }
 }
 
-void printFileListing() {
+void printFileListing(bool pretty) {
   File root;
   root = SD.open("/");
 
-  printDirectory(root, 0,"");
+  if (pretty) {
+    printDirectory(root, 0,"",true,false); //tabbed with file sizes
+  }else{
+    printDirectory(root, 0,"",false,true); //full paths without size
+  }
 }
 
-void printDirectory(File dir, int numTabs,String parent) {
+
-  bool printSize=false;
+void printDirectory(File dir, int numTabs,String parent,bool printSize,bool printFullDirectoryNames) {
-  bool printFullDirectoryNames=true;
+  
 
   while (true) {
 
@@ -400,7 +424,7 @@ void printDirectory(File dir, int numTabs,String parent) {
       if (!printFullDirectoryNames) {
         Serial.println("/");
       }
-      printDirectory(entry, numTabs + 1,parent+entry.name()+"/");
+      printDirectory(entry, numTabs + 1,parent+entry.name()+"/",printSize,printFullDirectoryNames);
     } else {
       if (printSize) {
         // files have sizes, directories do not
@@ -413,6 +437,17 @@ void printDirectory(File dir, int numTabs,String parent) {
   }
 }
 
+void printFilesize(String filename) {
+  File dataFile = SD.open(filename.c_str(), FILE_READ);
+  if (dataFile) {
+    Serial.println(dataFile.size(), DEC);
+    dataFile.close();
+  }else{
+    Serial.println('0');
+  }
+
+}
+
 
 void printFile(String filename) {
 
@@ -435,6 +470,52 @@ void printFile(String filename) {
 }
 
 
+void transferFile(String filename) {
+  //similar to printFile, but with checksum and chunk based transfer
+  
+  
+  
+
+  File dataFile = SD.open(filename.c_str(), FILE_READ);
+
+  bool transmiterror=false;
+
+  // if the file is available, write to it:
+  if (dataFile) {
+    while (dataFile.available() && !transmiterror) {
+      byte checksum=0;
+      for (uint16_t i=0;i<chunksize;i++) {
+        if (dataFile.available()) {
+          byte b = dataFile.read();
+          Serial.write(b);
+          checksum+=b;
+        }
+      }
+      int inByte=-1;
+      while (inByte==-1) {
+        if (Serial.available() > 0)
+        {
+          inByte = Serial.read();
+          
+          if ((byte)inByte!=checksum){
+            transmiterror=true; //exit
+            Serial.println();
+            Serial.println("TRANSMISSION ERROR WRONG CHECKSUM");
+          }
+        }
+      }
+
+    }
+    dataFile.close();
+  }
+  // if the file isn't open, pop up an error:
+  else {
+    Serial.print("error opening "); Serial.println(filename);
+  }
+  
+}
+
+
 void removeFile(String filename) {
 
   SD.remove(filename.c_str());

controller_teensy/src/main.cpp

@@ -99,6 +99,9 @@ void setup()
   bool initResult=false;
   
   initResult=display_init();
+  if (!initResult) {
+    writeLogComment((unsigned long)millis(), "SSD1306 allocation failed");
+  }
   led_simpeProgress(1,initResult);
 
   

logdata_visualization/copyLogsFromBobbycar.py

@@ -1,8 +1,17 @@
 
 import serial
 import os
+import time 
+chunksize=4096
 
-serialport = serial.Serial(port='/dev/ttyACM0', baudrate=115200, timeout=0.1)
+#LOG0002   
+#32 = 124 kbps
+#128 = 263 kbps
+#4096 = 416 kbps
+#32768 = 427 kbps
+#65536 = failed
+
+serialport = serial.Serial(port='/dev/ttyACM0', baudrate=115200, timeout=1)
 
 def establish_connection():
 
@@ -39,14 +48,38 @@ def get_filenames():
     return filenames
 
 
-def copy_file(source,destination):
+def get_filesize(filename):
-    os.makedirs(os.path.dirname(writefilename), exist_ok=True)
+
+    filesize=0
+
+    serialport.write(("sizeof "+str(filename)+"\n").encode())
+    
+    response = serialport.readline()
+    hrresponse=response.rstrip().decode('ascii')
+    if(len(response))>0:
+        filesize=int(hrresponse)
         
+    return filesize
+
+
+def copy_file(source,destination,expectedsize):
+    os.makedirs(os.path.dirname(writefilename), exist_ok=True)
+
+            
+    transferstarttime=time.time()
+    chunkstarttime=time.time()
+    last_print=time.time()
+
     with open(writefilename, 'wb') as writer:
-        serialport.write(("cat "+filename+"\n").encode())
+        serialport.write(("chunksize "+str(chunksize)+"\n").encode())
+        serialport.write(("get "+filename+"\n").encode())
                     
+
+        acc_datalen=0
+
         while True:
             
+            '''
             response = serialport.readline()
             hrresponse=response.rstrip().decode('ascii')
             if(len(response))>0:
@@ -55,6 +88,45 @@ def copy_file(source,destination):
                 
             else:
                 break
+            '''
+
+            
+            data=serialport.read(chunksize)
+            
+            if(len(data))>0: #data received
+                #print("received "+str(len(data))+" bytes")
+                #hrresponse=data.rstrip().decode('ascii')
+                #print(hrresponse)
+
+                acc_datalen+=len(data)
+
+                checksum=(sum(data) & 0xFF)
+                checksumarray=bytearray([checksum])
+
+                writer.write(data)
+                serialport.write(checksumarray) #request next chunk by sending checksum of last chunk        
+                
+            else:
+                break
+
+
+            if (time.time()-last_print>0.5):
+                last_print=time.time()
+                chunkduration=time.time()-chunkstarttime
+                chunkstarttime=time.time()
+                progress=acc_datalen/expectedsize
+                print(str(round(progress*100,0))+"% \t"+str(round(chunkduration*1000,3))+" ms for "+str(len(data))+" Byte \t = "+str(round((len(data)/chunkduration)/1000,3))+" kB/s")
+
+
+    fileduration=time.time()-transferstarttime
+        
+    file_stats=os.stat(writefilename)
+    print("Finished transfer of "+str(acc_datalen)+" B or "+str(file_stats.st_size)+" (os) Byte in "+str(fileduration)+" s \t = "+str(round(file_stats.st_size/fileduration/1000,3))+" kB/s") 
+
+
+    return acc_datalen
+
+
 
 
 def log_off():
@@ -87,10 +159,24 @@ if serialport.isOpen():
 
 
     #Copy all Files    
+    failed=0
+    
     for filename in filenames:
         print("Reading file "+filename)
+        expectedsize=get_filesize(filename)
+        print("Expecting "+str(expectedsize)+" Byte")
+
         writefilename='sdcard/'+filename
-        copy_file(filename,writefilename)
+        receivedsize=copy_file(filename,writefilename,expectedsize)
+        
+        if (expectedsize!=receivedsize):
+            print("Warning: Filesize does not match!")
+            failed+=1
+
+        print("")
+
+    print(str(len(filenames))+" Files copied with "+str(failed)+" failed")
+    
     
 
     #Delete all files
@@ -108,4 +194,5 @@ serialport.write("echo on\n".encode())
 
     
 
-serialport.close()
+serialport.close()
+