log and sorting works

grc/crfa_tmc_parser.xml

@@ -9,13 +9,21 @@
 #if not $label()
 	#set $label = '"%s"'%$id
 #end if
-$(parser) = crfa.tmc_parser($workdir, $log, $debug, $writeDB)
+$(parser) = crfa.tmc_parser($workdir, $log, $debug, $writeDB,maxheight)
 $(win) = $(parser).getqtwidget()
 $(gui_hint()($win))
   </make>
 <!--
   crfa.tmc_parser($workdir, $log, $debug, $writeDB)
 -->
+  <param>
+    <name>maxheight</name>
+    <key>maxheight</key>
+    <value>160</value>
+    <type>int</type>
+    <hide>part</hide>
+  </param>
+  
     <param>
     <name>work directory</name>
     <key>workdir</key>

python/rds_parser_table_qt.py

@@ -26,7 +26,7 @@ import pmt,functools,csv,md5,collections,copy,sqlite3,atexit,time,re,sys
 from datetime import datetime
 from datetime import timedelta
 import crfa.chart as chart
-from crfa.tmc_classes import tmc_dict,tmc_message
+from crfa.tmc_classes import tmc_dict,tmc_message,language
 
 from PyQt4 import Qt, QtCore, QtGui
 import pprint,code,pickle#for easier testing
@@ -39,7 +39,7 @@ pr = cProfile.Profile()
 from PyQt4.QtCore import QObject, pyqtSignal
 from bitstring import BitArray
 
-language="de"#currently supported: de, en (both partially)
+#language="de"#currently supported: de, en (both partially) #defined in tmc_classes.py
 
 
 class rds_parser_table_qt_Signals(QObject):
@@ -627,6 +627,7 @@ class rds_parser_table_qt(gr.sync_block):#START
                         print("unknown variant %i in TMC 3A group"%variant)
                     send_pmt = pmt.pmt_to_python.pmt_from_dict({
                         "type":"3A_meta",
+                        "PI":PI,
                         "data":self.RDS_data[PI]["AID_list"][AID]})
                     self.message_port_pub(pmt.intern('tmc_raw'), send_pmt)
             elif (groupType == "4A"):#CT clock time
@@ -684,10 +685,17 @@ class rds_parser_table_qt(gr.sync_block):#START
                 tmc_x=array[3]&0x1f #lower 5 bit of block2
                 tmc_y=(array[4]<<8)|(array[5]) #block3
                 tmc_z=(array[6]<<8)|(array[7])#block4
+                datetime_received=self.RDS_data[PI]["time"]["datetime"]
+                psn=self.RDS_data[PI]["PSN"]
+                if datetime_received==None:
+                    datetime_str=""
+                else:
+                    datetime_str=datetime_received.strftime("%Y-%m-%d %H:%M:%S")
                 send_pmt = pmt.pmt_to_python.pmt_from_dict({
                     "type":"alert-c",
                     "PI":PI,
-                    "PSN":self.RDS_data[PI]["PSN"],
+                    "PSN":psn,
+                    "datetime_str":datetime_str,
                     "TMC_X":tmc_x,
                     "TMC_Y":tmc_y,
                     "TMC_Z":tmc_z
@@ -697,14 +705,20 @@ class rds_parser_table_qt(gr.sync_block):#START
                 tmc_T=tmc_x>>4 #0:TMC-message 1:tuning info/service provider name
                 tmc_F=int((tmc_x>>3)&0x1) #identifies the message as a Single Group (F = 1) or Multi Group (F = 0)
                 Y15=int(tmc_y>>15)
-                datetime_received=self.RDS_data[PI]["time"]["datetime"]
+                try:
+                    ltn=self.RDS_data[PI]["AID_list"][52550]["LTN"]
+                except KeyError:
+                    ltn=1#assume germany TODO:add better error handling
+                if self.log:
+                    print("no LTN (yet) for PI:%s"%PI)
                 if tmc_T == 0:
                     if tmc_F==1:#single group
-                        tmc_msg=tmc_message(PI,tmc_x,tmc_y,tmc_z,datetime_received,self)
+                        
+                        tmc_msg=tmc_message(PI,psn,ltn,tmc_x,tmc_y,tmc_z,datetime_received,self)
                         self.print_tmc_msg(tmc_msg)
                     elif tmc_F==0 and Y15==1:#1st group of multigroup
                         ci=int(tmc_x&0x7)
-                        tmc_msg=tmc_message(PI,tmc_x,tmc_y,tmc_z,datetime_received,self)
+                        tmc_msg=tmc_message(PI,psn,ltn,tmc_x,tmc_y,tmc_z,datetime_received,self)
                         #if  self.RDS_data[PI]["internals"]["unfinished_TMC"].has_key(ci):
                             #print("overwriting parital message")
                         self.RDS_data[PI]["internals"]["unfinished_TMC"][ci]={"msg":tmc_msg,"time":time.time()}
@@ -731,7 +745,11 @@ class rds_parser_table_qt(gr.sync_block):#START
                         #seen variants 4569, 6 most often
                         #print("TMC-info variant:%i"%adr)
                         if adr==4 or adr==5:#service provider name
-                            segment=self.decode_chars(chr(array[4])+chr(array[5])+chr(array[6])+chr(array[7]))
+                            chr1=(tmc_y >> 8) & 0xff
+                            chr2=tmc_y & 0xff
+                            chr3=(tmc_z >> 8) & 0xff
+                            chr4=tmc_z & 0xff
+                            segment=self.decode_chars(chr(chr1)+chr(chr2)+chr(chr3)+chr(chr4))
                             if self.debug:
                                 print("TMC-info adr:%i (provider name), segment:%s, station:%s"%(adr,segment,self.RDS_data[PI]["PSN"]))
                             if self.RDS_data[PI]["AID_list"].has_key(52550):

python/tmc_classes.py

@@ -18,6 +18,16 @@
 # the Free Software Foundation, Inc., 51 Franklin Street,
 # Boston, MA 02110-1301, USA.
 # 
+
+#references to tableobj:
+#~ label6_suppl_info
+#~ ecl_dict
+#~ tmc_update_class_names
+#~ lcl_dict
+#~ debug
+#~ tmc_messages
+
+#rename to common.py?
 from bitstring import BitArray 
 import copy
 
@@ -496,11 +506,12 @@ class tmc_message:
       return self.datetime_received.strftime("%H:%M")
     else:
       return "88:88"
-  def __init__(self,PI,tmc_x,tmc_y,tmc_z,datetime_received,tableobj):#TODO handle out of sequence data
+  def __init__(self,PI,psn,ltn,tmc_x,tmc_y,tmc_z,datetime_received,tableobj):#TODO handle out of sequence data
-    self.psn=tableobj.RDS_data[PI]["PSN"]
+    #self.psn=tableobj.RDS_data[PI]["PSN"]
+    self.psn=psn
     #check LTN
     try:
-      msg_ltn=tableobj.RDS_data[PI]["AID_list"][52550]["LTN"]
+      msg_ltn=ltn#tableobj.RDS_data[PI]["AID_list"][52550]["LTN"]
       table_ltn=1#german table
       if msg_ltn != table_ltn  and tableobj.debug and False:#disabled, spams log
 	print("msg_ltn:%i does not match expected table (1) on station: %s"%(msg_ltn,self.psn))

python/tmc_parser.py

@@ -23,31 +23,197 @@ import numpy
 from gnuradio import gr
 import pmt
 from PyQt4 import Qt, QtCore, QtGui
+import code,time,csv,sqlite3,atexit
+from bitstring import BitArray
+from crfa.tmc_classes import tmc_dict,tmc_message,language
+from datetime import datetime
+from datetime import timedelta
 
 class tmc_parser(gr.sync_block):
     """
     docstring for block tmc_parser
     """
-    def __init__(self, workdir,log,debug,writeDB):
+    def __init__(self, workdir,log,debug,writeDB,maxheight):
         gr.sync_block.__init__(self,
             name="tmc_parser",
             in_sig=None,
             out_sig=None)
+        self.log=log
+        self.debug=debug
+        self.workdir=workdir
+        atexit.register(self.goodbye)
+        self.qtwidget=tmc_parser_Widget(self,maxheight)
         self.message_port_register_in(pmt.intern('in'))
         self.set_msg_handler(pmt.intern('in'), self.handle_msg)
-        self.qtwidget=tmc_parser_Widget(self)
+        self.tmc_meta={}
+        self.unfinished_messages={}
+        self.TMC_data={}
+        self.tmc_messages=tmc_dict()
+        reader = csv.reader(open(self.workdir+'LCL15.1.D-160122_utf8.csv'), delimiter=';', quotechar='"')
+        reader.next()#skip header
+        self.lcl_dict=dict((int(rows[0]),rows[1:]) for rows in reader)
+        #read TMC-event list
+        reader = csv.reader(open(self.workdir+'event-list_with_forecast_sort.csv'), delimiter=',', quotechar='"')
+        reader.next()#skip header
+        self.ecl_dict=dict((int(rows[0]),rows[1:]) for rows in reader)
+        #read supplementary information code list
+        reader = csv.reader(open(self.workdir+'label6-supplementary-information-codes.csv'), delimiter=',', quotechar='"')
+        reader.next()#skip header, "code,english,german"
+        if language=="de":
+            self.label6_suppl_info=dict((int(rows[0]),rows[2]) for rows in reader)#german
+        else:
+            self.label6_suppl_info=dict((int(rows[0]),rows[1]) for rows in reader)#english
+        #read update classes
+        reader = csv.reader(open(self.workdir+'tmc_update_class_names.csv'), delimiter=',', quotechar='"')
+        reader.next()#skip header, "code(C),english,german"
+        if language=="de":
+            self.tmc_update_class_names=dict((int(rows[0]),rows[2]) for rows in reader)#german names
+        else:
+            self.tmc_update_class_names=dict((int(rows[0]),rows[1]) for rows in reader)#english names  
+    def goodbye(self):
+        print("closing tmc display")
+    def print_tmc_msg(self,tmc_msg):
+        self.qtwidget.print_tmc_msg(tmc_msg)
+        if self.debug:
+            print("new tmc message %s"%tmc_msg)
+    def initialize_data_for_PI(self,PI):
+        self.unfinished_messages[PI]={}
     def handle_msg(self,msg):
         m=pmt.to_python(msg)
-        self.qtwidget.updateui()
+        PI=m["PI"]
-        print(m)
+        if not self.unfinished_messages.has_key(PI):
+            self.initialize_data_for_PI(PI)
+        if m["type"]=="3A_meta":
+            self.tmc_meta[PI]=m["data"]
+        elif m["type"]=="alert-c":
+            #self.qtwidget.updateui()
+            #print(m)
+            psn=m["PSN"]
+            try:
+                ltn=self.tmc_meta[PI]["LTN"]
+            except KeyError:
+                ltn=1#assume germany TODO:add better error handling
+                if self.log:
+                    print("no LTN (yet) for PI:%s"%PI)
+            tmc_x=m["TMC_X"]
+            tmc_y=m["TMC_Y"]
+            tmc_z=m["TMC_Z"]
+            if m["datetime_str"]=="":
+                datetime_received=None
+            else:
+                datetime_received=datetime.strptime(m["datetime_str"],"%Y-%m-%d %H:%M:%S")
+            tmc_T=tmc_x>>4 #0:TMC-message 1:tuning info/service provider name
+            tmc_F=int((tmc_x>>3)&0x1) #identifies the message as a Single Group (F = 1) or Multi Group (F = 0)
+            Y15=int(tmc_y>>15)
+            if tmc_T == 0:
+                if tmc_F==1:#single group
+                    tmc_msg=tmc_message(PI,psn,ltn,tmc_x,tmc_y,tmc_z,datetime_received,self)
+                    self.print_tmc_msg(tmc_msg)
+                elif tmc_F==0 and Y15==1:#1st group of multigroup
+                    ci=int(tmc_x&0x7)
+                    tmc_msg=tmc_message(PI,psn,ltn,tmc_x,tmc_y,tmc_z,datetime_received,self)
+                    #if  self.RDS_data[PI]["internals"]["unfinished_TMC"].has_key(ci):
+                        #print("overwriting parital message")
+                    self.unfinished_messages[PI][ci]={"msg":tmc_msg,"time":time.time()}
+                else:
+                    ci=int(tmc_x&0x7)
+                    if self.unfinished_messages[PI].has_key(ci):
+                        tmc_msg=self.unfinished_messages[PI][ci]["msg"]
+                        tmc_msg.add_group(tmc_y,tmc_z)
+                        age=time.time()-self.unfinished_messages[PI][ci]["time"]
+                        t=(time.time(),PI,age,ci,tmc_msg.is_complete)
+                        #print("%f: continuing message PI:%s,age:%f,ci:%i complete:%i"%t)
+                        self.unfinished_messages[PI]["time"]=time.time()
+                        if tmc_msg.is_complete:
+                            self.print_tmc_msg(tmc_msg)#print and store message
+                            del self.unfinished_messages[PI][tmc_msg.ci]#delete finished message
+                    else:
+                        #if not ci==0:
+                            #print("ci %i not found, discarding"%ci)
+                        pass
+
+            else:#alert plus or provider info
+                adr=tmc_x&0xf
+                if  4 <= adr and adr <= 9:
+                    #seen variants 4569, 6 most often
+                    #print("TMC-info variant:%i"%adr)
+                    if adr==4 or adr==5:#service provider name
+                        chr1=(tmc_y >> 8) & 0xff
+                        chr2=tmc_y & 0xff
+                        chr3=(tmc_z >> 8) & 0xff
+                        chr4=tmc_z & 0xff
+                        segment=self.decode_chars(chr(chr1)+chr(chr2)+chr(chr3)+chr(chr4))
+                        if self.log:
+                            print("TMC-info adr:%i (provider name), segment:%s, station:%s"%(adr,psn))
+                    if adr== 7:#freq of tuned an mapped station (not seen yet)
+                        freq_TN=tmc_y>>8
+                        freq_ON=tmc_y&0xff#mapped frequency
+                        if self.log:
+                            print("TMC-info: TN:%i, station:%s"%(freq_TN,psn))
+                else:
+                    if self.log:
+                        print("alert plus on station %s (%s)"%(PI,psn))#(not seen yet)
+            
     def getqtwidget(self):
         return self.qtwidget
+    def decode_chars(self,charstring):
+        alphabet={
+        0b0010:u" !\#¤%&'()*+,-./",
+        0b0011:u"0123456789:;<=>?",
+        0b0100:u"@ABCDEFGHIJKLMNO",
+        0b0101:u"PQRSTUVWXYZ[\]―_",
+        0b0110:u"‖abcdefghijklmno",
+        0b0111:u"pqrstuvwxyz{|}¯ ",
+        0b1000:u"áàéèíìóòúùÑÇŞßiIJ",
+        0b1001:u"âäêëîïôöûüñçşǧıij",
+        0b1010:u"ªα©‰Ǧěňőπ€£$←↑→↓",
+        0b1011:u"º¹²³±İńűµ¿÷°¼½¾§",
+        0b1100:u"ÁÀÉÈÍÌÓÒÚÙŘČŠŽĐĿ",
+        0b1101:u"ÂÄÊËÎÏÔÖÛÜřčšžđŀ",
+        0b1110:u"ÃÅÆŒŷÝÕØÞŊŔĆŚŹŦð",
+        0b1111:u"ãåæœŵýõøþŋŕćśźŧ "}#0xff should not occur (not in standard) (but occured 2017-03-04-9:18 , probably transmission error)
+
+        #charlist=list(charstring)
+        return_string=""
+        for i,char in enumerate(charstring):
+            #split byte
+            alnr=(ord(char)&0xF0 )>>4 #upper 4 bit
+            index=ord(char)&0x0F #lower 4 bit
+            if ord(char)<= 0b00011111:#control code
+                if ord(char)==0x0D or ord(char)==0x00:#end of message SWR uses: \r\0\0\0 for last block (\0 fill 4 char segment)
+                    #return_string+="\r"
+                    return_string+=char
+                else:
+                    return_string+="{%02X}"%ord(char)#output control code
+#       elif ord(char)<= 0b01111111: #real encoding slightly different from ascii
+#         return_string+=char#use ascii
+            else:
+                try:
+                    return_string+=alphabet[alnr][index]
+                    #return_string+=unichr(ord(char))#TODO: properly decide for UTF8 or EBU charset
+                except KeyError:
+                    return_string+="?%02X?"%ord(char)#symbol not decoded
+                    print("symbol not decoded: "+"?%02X?"%ord(char)+"in string:"+return_string)
+                    pass
+        return return_string
 class tmc_parser_Widget(QtGui.QWidget):
+    def print_tmc_msg(self,tmc_msg):
+        ef=unicode(self.event_filter.text().toUtf8(), encoding="UTF-8").lower()
+        lf=unicode(self.location_filter.text().toUtf8(), encoding="UTF-8").lower()
+        filters=[{"type":"location", "str":lf},{"type":"event", "str":ef}]
+        if self.parser.tmc_messages.matchFilter(tmc_msg,filters):
+            self.logOutput.append(Qt.QString.fromUtf8(tmc_msg.log_string()))
+            self.logOutput.append(Qt.QString.fromUtf8(tmc_msg.multi_str()))
     def updateui(self):
         print("updating ui")
     def filterChanged(self):
+        ef=unicode(self.event_filter.text().toUtf8(), encoding="UTF-8").lower()
+        lf=unicode(self.location_filter.text().toUtf8(), encoding="UTF-8").lower()
+        self.logOutput.clear()
+        filters=[{"type":"location", "str":lf},{"type":"event", "str":ef}]
+        self.logOutput.append(Qt.QString.fromUtf8(self.parser.tmc_messages.getLogString(filters)))
         print("filter changed")
-    def __init__(self, parser):
+    def __init__(self, parser,maxheight):
         QtGui.QWidget.__init__(self)
         layout = Qt.QVBoxLayout()
         self.setLayout(layout)
@@ -69,7 +235,8 @@ class tmc_parser_Widget(QtGui.QWidget):
         self.logOutput = Qt.QTextEdit()
         self.logOutput.setReadOnly(True)
         self.logOutput.setLineWrapMode(Qt.QTextEdit.NoWrap)
-        self.logOutput.setMaximumHeight(150)
+        #self.logOutput.setMaximumHeight(150) #label was too wide
+        self.setMaximumHeight(maxheight)
         font = self.logOutput.font()
         font.setFamily("Courier")
         font.setPointSize(10)