csrichter
/
gr-multirds
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity
Browse Source

db saving and marker generation moved to tmc_parser

master
Clemens Richter 9 years ago
parent
aeaeebcfbc
commit
c43d6642a5
5 changed files with 111 additions and 154 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Split View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 16
      grc/crfa_rds_parser_table_qt.xml
  2. 4
      grc/crfa_tmc_parser.xml
  3. 167
      python/rds_parser_table_qt.py
  4. 7
      python/tmc_classes.py
  5. 45
      python/tmc_parser.py

16
grc/crfa_rds_parser_table_qt.xml
Unescape View File

@ -14,7 +14,7 @@
#set $label = '"%s"'%$id
#end if
$(signals) = rds_parser_table_qt_Signals()
self.$(id) = crfa.rds_parser_table_qt($(signals),$nPorts,self.set_$(freq_tune),$freq_tune,$log, $debug,$workdir,$writeDB,$showTMC)
self.$(id) = crfa.rds_parser_table_qt($(signals),$nPorts,self.set_$(freq_tune),$freq_tune,$log, $debug,$workdir,$writeDB)
$(win) = rds_parser_table_qt_Widget($signals, $label,self.$(id),$showTMC)
$(gui_hint()($win))</make>
<callback>set_freq_tune($freq_tune);</callback>
@ -51,20 +51,6 @@ $(gui_hint()($win))</make>
<value>2</value>
<type>int</type>
<hide>part</hide>
</param>
<param>
<name>show TMC</name>
<key>showTMC</key>
<value>False</value>
<type>bool</type>
<option>
<name>Enable</name>
<key>True</key>
</option>
<option>
<name>Disable</name>
<key>False</key>
</option>
</param>
<param>
<name>Log</name>

4
grc/crfa_tmc_parser.xml
Unescape View File

@ -105,4 +105,8 @@ check if event-list file exists
<type>message</type>
<optional>1</optional>
</sink>
<doc>
maxheight = max height of widget in pixels
maxheight == 0 -> no restriction
</doc>
</block>

167
python/rds_parser_table_qt.py
Unescape View File

@ -56,7 +56,7 @@ class rds_parser_table_qt(gr.sync_block):#START
if self.writeDB:
#self.db.commit()
self.db.close()
def __init__(self,signals,nPorts,slot,freq,log,debug,workdir,writeDB,showTMC):
def __init__(self,signals,nPorts,slot,freq,log,debug,workdir,writeDB):
gr.sync_block.__init__(self,
name="RDS Table",
in_sig=None,
@ -77,7 +77,6 @@ class rds_parser_table_qt(gr.sync_block):#START
self.log=log
self.debug=debug
self.writeDB=writeDB
self.showTMC=showTMC
self.signals=signals
self.RDS_data={}
self.change_freq_tune=slot
@ -112,8 +111,8 @@ class rds_parser_table_qt(gr.sync_block):#START
(time text,PI text,PSN text, dataType text,data blob)''')
db.execute('''CREATE TABLE grouptypeCounts
(PI text,grouptype text,count integer,unique (PI, grouptype))''')
db.execute('''CREATE TABLE TMC
(hash text PRIMARY KEY UNIQUE,time text,PI text, F integer,event integer,location integer,DP integer,div integer,dir integer,extent integer,text text,multi text,rawmgm text)''')
# db.execute('''CREATE TABLE TMC
# (hash text PRIMARY KEY UNIQUE,time text,PI text, F integer,event integer,location integer,DP integer,div integer,dir integer,extent integer,text text,multi text,rawmgm text)''')
db.commit()
except sqlite3.OperationalError:
@ -178,12 +177,6 @@ class rds_parser_table_qt(gr.sync_block):#START
#print(self.PI_dict)
if self.writeDB:
self.db.commit()
f=open(self.workdir+'google_maps_markers.js', 'w')
markerstring=self.tmc_messages.getMarkerString()
markerstring+='\n console.log("loaded "+markers.length+" markers")'
markerstring+='\n document.getElementById("errorid").innerHTML = "loaded "+markers.length+" markers";'
f.write(markerstring)
f.close()
def update_freq(self):
# "&#9;" is a tab character
message_string="decoder frequencies:"
@ -701,49 +694,51 @@ class rds_parser_table_qt(gr.sync_block):#START
"TMC_Z":tmc_z
})
self.message_port_pub(pmt.intern('tmc_raw'), send_pmt)
tmc_hash=md5.new(str([PI,tmc_x,tmc_y,tmc_z])).hexdigest()
tmc_T=tmc_x>>4 #0:TMC-message 1:tuning info/service provider name
tmc_F=int((tmc_x>>3)&0x1) #identifies the message as a Single Group (F = 1) or Multi Group (F = 0)
Y15=int(tmc_y>>15)
try:
ltn=self.RDS_data[PI]["AID_list"][52550]["LTN"]
except KeyError:
ltn=1#assume germany TODO:add better error handling
if self.log:
print("no LTN (yet) for PI:%s"%PI)
if tmc_T == 0:
if tmc_F==1:#single group
tmc_msg=tmc_message(PI,psn,ltn,tmc_x,tmc_y,tmc_z,datetime_received,self)
self.print_tmc_msg(tmc_msg)
elif tmc_F==0 and Y15==1:#1st group of multigroup
ci=int(tmc_x&0x7)
tmc_msg=tmc_message(PI,psn,ltn,tmc_x,tmc_y,tmc_z,datetime_received,self)
#if self.RDS_data[PI]["internals"]["unfinished_TMC"].has_key(ci):
#print("overwriting parital message")
self.RDS_data[PI]["internals"]["unfinished_TMC"][ci]={"msg":tmc_msg,"time":time.time()}
else:
ci=int(tmc_x&0x7)
if self.RDS_data[PI]["internals"]["unfinished_TMC"].has_key(ci):
tmc_msg=self.RDS_data[PI]["internals"]["unfinished_TMC"][ci]["msg"]
tmc_msg.add_group(tmc_y,tmc_z)
age=time.time()-self.RDS_data[PI]["internals"]["unfinished_TMC"][ci]["time"]
t=(time.time(),PI,age,ci,tmc_msg.is_complete)
#print("%f: continuing message PI:%s,age:%f,ci:%i complete:%i"%t)
self.RDS_data[PI]["internals"]["unfinished_TMC"][ci]["time"]=time.time()
if tmc_msg.is_complete:
self.print_tmc_msg(tmc_msg)#print and store message
del self.RDS_data[PI]["internals"]["unfinished_TMC"][tmc_msg.ci]#delete finished message
else:
#if not ci==0:
#print("ci %i not found, discarding"%ci)
pass
else:#alert plus or provider info
#~ tmc_hash=md5.new(str([PI,tmc_x,tmc_y,tmc_z])).hexdigest()
#~ tmc_T=tmc_x>>4 #0:TMC-message 1:tuning info/service provider name
#~ tmc_F=int((tmc_x>>3)&0x1) #identifies the message as a Single Group (F = 1) or Multi Group (F = 0)
#~ Y15=int(tmc_y>>15)
#~ try:
#~ ltn=self.RDS_data[PI]["AID_list"][52550]["LTN"]
#~ except KeyError:
#~ ltn=1#assume germany TODO:add better error handling
#~ if self.log:
#~ print("no LTN (yet) for PI:%s"%PI)
#~ if tmc_T == 0:
#~ if tmc_F==1:#single group
#~ tmc_msg=tmc_message(PI,psn,ltn,tmc_x,tmc_y,tmc_z,datetime_received,self)
#~ self.print_tmc_msg(tmc_msg)
#~ elif tmc_F==0 and Y15==1:#1st group of multigroup
#~ ci=int(tmc_x&0x7)
#~ tmc_msg=tmc_message(PI,psn,ltn,tmc_x,tmc_y,tmc_z,datetime_received,self)
#~ #if self.RDS_data[PI]["internals"]["unfinished_TMC"].has_key(ci):
#~ #print("overwriting parital message")
#~ self.RDS_data[PI]["internals"]["unfinished_TMC"][ci]={"msg":tmc_msg,"time":time.time()}
#~ else:
#~ ci=int(tmc_x&0x7)
#~ if self.RDS_data[PI]["internals"]["unfinished_TMC"].has_key(ci):
#~ tmc_msg=self.RDS_data[PI]["internals"]["unfinished_TMC"][ci]["msg"]
#~ tmc_msg.add_group(tmc_y,tmc_z)
#~ age=time.time()-self.RDS_data[PI]["internals"]["unfinished_TMC"][ci]["time"]
#~ t=(time.time(),PI,age,ci,tmc_msg.is_complete)
#~ #print("%f: continuing message PI:%s,age:%f,ci:%i complete:%i"%t)
#~ self.RDS_data[PI]["internals"]["unfinished_TMC"][ci]["time"]=time.time()
#~ if tmc_msg.is_complete:
#~ self.print_tmc_msg(tmc_msg)#print and store message
#~ del self.RDS_data[PI]["internals"]["unfinished_TMC"][tmc_msg.ci]#delete finished message
#~ else:
#~ #if not ci==0:
#~ #print("ci %i not found, discarding"%ci)
#~ pass
#~ else:#alert plus or provider info
if tmc_T == 1:#rest done by tmc_parser
adr=tmc_x&0xf
if 4 <= adr and adr <= 9:
#seen variants 4569, 6 most often
#print("TMC-info variant:%i"%adr)
#~ #print("TMC-info variant:%i"%adr)
if adr==4 or adr==5:#service provider name
chr1=(tmc_y >> 8) & 0xff
chr2=tmc_y & 0xff
@ -757,6 +752,7 @@ class rds_parser_table_qt(gr.sync_block):#START
seg_adr_start=(adr-4)*4#start of segment
text_list[seg_adr_start:seg_adr_start+4]=segment
self.RDS_data[PI]["AID_list"][52550]["provider name"]="".join(text_list)
if adr== 7:#freq of tuned an mapped station (not seen yet)
freq_TN=tmc_y>>8
freq_ON=tmc_y&0xff#mapped frequency
@ -764,10 +760,10 @@ class rds_parser_table_qt(gr.sync_block):#START
print("TMC-info: TN:%i, station:%s"%(freq_TN,self.RDS_data[PI]["PSN"]))
self.RDS_data[PI]["TMC_TN"]=freq_TN
else:
if self.debug:
if self.log or self.debug:
print("alert plus on station %s (%s)"%(PI,self.RDS_data[PI]["PSN"]))#(not seen yet)
#self.tableobj.RDS_data["D301"]["AID_list"][52550]["provider name"]="test____"
#~ #self.tableobj.RDS_data["D301"]["AID_list"][52550]["provider name"]="test____"
#RadioText+ (grouptype mostly 12A):
elif self.RDS_data[PI]["AID_list"].has_key(19415) and self.RDS_data[PI]["AID_list"][19415]["groupType"]==groupType:#RT+
if not self.RDS_data[PI].has_key("RT+"):
@ -939,54 +935,9 @@ class rds_parser_table_qt(gr.sync_block):#START
print("8A without 3A on PI:%s"%PI)
#else:#other group
#print("group of type %s not decoded on station %s"% (groupType,PI))
if 1==1:
#printdelay=50
printdelay=500
self.printcounter+=0#printing disabled
if self.printcounter == printdelay and self.debug:
for key in self.RDS_data:
if self.RDS_data[key].has_key("PSN"):
psn=self.RDS_data[key]["PSN"]
else:
psn="?"
print("%s(%s):"%(psn,key),end="")
pp.pprint(self.RDS_data[key]["blockcounts"])
if self.RDS_data[key].has_key("RT+"):
print("RT+:",end="")
pp.pprint(self.RDS_data[key]["RT+"])
self.printcounter=0
pr.disable() #disabled-internal-profiling
#end of handle_msg
def print_tmc_msg(self,tmc_msg):
try:
PI=tmc_msg.PI
tmc_F=tmc_msg.is_single
tmc_hash=tmc_msg.tmc_hash
refloc_name=""
reflocs=tmc_msg.location.reflocs
if not self.TMC_data.has_key(tmc_hash):#if message new
try:
self.TMC_data[tmc_hash]=tmc_msg
if self.showTMC:
self.signals.DataUpdateEvent.emit({'TMC_log':tmc_msg,'multi_str':tmc_msg.multi_str()})
#t=(str(datetime.now()),PI,self.RDS_data[PI]["PSN"],"ALERT-C",message_string.decode("utf-8"))
#self.db.execute("INSERT INTO data (time,PI,PSN,dataType,data) VALUES (?,?,?,?,?)",t)
timestring=self.RDS_data[PI]["time"]["timestring"]
if self.writeDB:
message_string=tmc_msg.db_string()
t=(tmc_hash,timestring,PI, tmc_F,tmc_msg.event.ecn,int(tmc_msg.location.lcn),tmc_msg.tmc_DP,tmc_msg.tmc_D,tmc_msg.tmc_dir,tmc_msg.tmc_extent,message_string.decode("utf-8"),tmc_msg.multi_str().decode("utf-8"),str(tmc_msg.debug_data))
self.db.execute("INSERT INTO TMC (hash,time,PI, F,event,location,DP,div,dir,extent,text,multi,rawmgm) VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?)",t)
except Exception as e:
print(e)
raise
#print("line 1064")
except KeyError:
#print("location '%i' not found"%tmc_location)
pass
def print_results(self):
s = StringIO.StringIO()
sortby = 'cumulative'
@ -1115,30 +1066,6 @@ class rds_parser_table_qt_Widget(QtGui.QWidget):
layout.addLayout(label_layout)
#TODO set different minsize if TMC is shown
self.setMinimumSize(Qt.QSize(500,40*self.tableobj.nPorts))
if self.showTMC:
self.tmc_message_label=QtGui.QLabel("TMC messages:")
self.event_filter=QtGui.QLineEdit()#QPlainTextEdit ?
self.location_filter=QtGui.QLineEdit(u"Baden-Württemberg")
#self.location_filter=QtGui.QLineEdit(u"")
self.event_filter.returnPressed.connect(self.filterChanged)
self.location_filter.returnPressed.connect(self.filterChanged)
filter_layout = Qt.QHBoxLayout()
filter_layout.addWidget(QtGui.QLabel("event filter:"))
filter_layout.addWidget(self.event_filter)
filter_layout.addWidget(QtGui.QLabel("location filter:"))
filter_layout.addWidget(self.location_filter)
layout.addLayout(filter_layout)
layout.addWidget(self.tmc_message_label)
self.logOutput = Qt.QTextEdit()
self.logOutput.setReadOnly(True)
self.logOutput.setLineWrapMode(Qt.QTextEdit.NoWrap)
self.logOutput.setMaximumHeight(150)
font = self.logOutput.font()
font.setFamily("Courier")
font.setPointSize(10)
layout.addWidget(self.logOutput)
self.lastResizeTime=0
self.clip = QtGui.QApplication.clipboard()
#self.cb.clear(mode=cb.Clipboard )

7
python/tmc_classes.py
Unescape View File

@ -51,6 +51,7 @@ class tmc_event:
self.name="##Error##"
self.length_str=None
self.speed_limit_str=None
self.is_cancellation = False
try:
#Code,Text CEN-English,Text (German),Text (German) kein Quantifier,Text (Quantifier = 1),Text (Quantifier >1),N,Q,T,D,U,C,R ,Comment
event_array=self.tableobj.ecl_dict[ecn]
@ -66,8 +67,6 @@ class tmc_event:
self.nature=event_array[5]#N:nature (blank): information, F:forecast, S:silent
if event_array[0]=="message cancelled":
self.is_cancellation = True
else:
self.is_cancellation = False
self.quantifierType=event_array[6]#Q:quantifier type: (0..12) or blank (no quantifier)
self.durationType=event_array[7]#T:duration type: D:dynamic, L:long lasting, in brackets or if time-of-day quantifier (no 7) is used in message -> no display, only for management
directionality=event_array[8]#D:directionality: 1:unidirectional, 2:bidirectional, cancellation messages dont have directionality
@ -445,8 +444,8 @@ class tmc_message:
def log_string(self):
return str(self.event.updateClass)+": "+self.getTime()+": "+self.location_text()+": "+self.events_string()+"; "+self.info_str()+"; "+self.psn.encode("utf-8")
#2017-03-16 fix:self.psn.encode("utf-8"), utf8 decoding happens later
def db_string(self):
return str(self.location)+": "+str(self.event.updateClass)+": "+self.events_string()+"; "+self.info_str()
#def db_string(self):
# return str(self.location)+": "+str(self.event.updateClass)+": "+self.events_string()+"; "+self.info_str()
def map_string(self):
return '<span title="%s">'%self.getDate()+str(self.event.updateClass)+": "+self.getTime()+'</span><span title="%s">'%self.multi_str()+": "+self.events_string()+self.info_str()+"; "+self.psn.encode("utf-8")+"</span>"
def end_loc(self):

45
python/tmc_parser.py
Unescape View File

@ -41,7 +41,7 @@ class tmc_parser(gr.sync_block):
self.log=log
self.debug=debug
self.workdir=workdir
atexit.register(self.goodbye)
self.writeDB=writeDB
self.qtwidget=tmc_parser_Widget(self,maxheight)
self.message_port_register_in(pmt.intern('in'))
self.set_msg_handler(pmt.intern('in'), self.handle_msg)
@ -49,6 +49,25 @@ class tmc_parser(gr.sync_block):
self.unfinished_messages={}
self.TMC_data={}
self.tmc_messages=tmc_dict()
atexit.register(self.goodbye)
if self.writeDB:
#create new DB file
db_name=workdir+'RDS_data'+datetime.now().strftime("%Y%m%d_%H%M%S")+'_TMC.db'
db=sqlite3.connect(db_name, check_same_thread=False)
self.db=db
#create tables
try:
#db.execute('CREATE TABLE TMC(hash text PRIMARY KEY UNIQUE,time text,PI text, F integer,event integer,location integer,DP integer,div integer,dir integer,extent integer,text text,multi text,rawmgm text)')
db.execute('''CREATE TABLE TMC(lcn integer,updateclass integer,
PI text,time text,ecn integer, isSingle integer,DP integer,div integer,dir integer,extent integer,
locstr text,eventstr text,multistr text,infostr text,
PRIMARY KEY (lcn, updateclass,PI))''')
db.commit()
except sqlite3.OperationalError as e:
print("ERROR: tables already exist")
print(e)
reader = csv.reader(open(self.workdir+'LCL15.1.D-160122_utf8.csv'), delimiter=';', quotechar='"')
reader.next()#skip header
self.lcl_dict=dict((int(rows[0]),rows[1:]) for rows in reader)
@ -71,8 +90,30 @@ class tmc_parser(gr.sync_block):
else:
self.tmc_update_class_names=dict((int(rows[0]),rows[1]) for rows in reader)#english names
def goodbye(self):
self.save_data()
print("closing tmc display")
def save_data(self):
if self.writeDB:
self.db.commit()
f=open(self.workdir+'google_maps_markers.js', 'w')
markerstring=self.tmc_messages.getMarkerString()
markerstring+='\n console.log("loaded "+markers.length+" markers")'
markerstring+='\n document.getElementById("errorid").innerHTML = "loaded "+markers.length+" markers";'
f.write(markerstring)
f.close()
def print_tmc_msg(self,tmc_msg):
if self.writeDB and tmc_msg.event.is_cancellation == False:
try:
t=(int(tmc_msg.location.lcn),int(tmc_msg.event.updateClass),tmc_msg.PI,
tmc_msg.getTime(),int(tmc_msg.event.ecn),int(tmc_msg.is_single),
int(tmc_msg.tmc_DP),int(tmc_msg.tmc_D),int(tmc_msg.tmc_dir),int(tmc_msg.tmc_extent),
tmc_msg.location_text().decode("utf-8"),tmc_msg.events_string().decode("utf-8"),tmc_msg.info_str().decode("utf-8"),tmc_msg.multi_str().decode("utf-8"))
self.db.execute("REPLACE INTO TMC (lcn,updateclass,PI,time,ecn,isSingle,DP,div,dir,extent,locstr,eventstr,infostr,multistr) VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?)",t)
except Exception as e:
if self.log or self.debug:
print("error during db insert msg:%s"%tmc_msg.log_string())
print(e)
pass
self.qtwidget.print_tmc_msg(tmc_msg)
if self.debug:
print("new tmc message %s"%tmc_msg)
@ -235,7 +276,7 @@ class tmc_parser_Widget(QtGui.QWidget):
self.logOutput = Qt.QTextEdit()
self.logOutput.setReadOnly(True)
self.logOutput.setLineWrapMode(Qt.QTextEdit.NoWrap)
#self.logOutput.setMaximumHeight(150) #label was too wide
if not maxheight==0:
self.setMaximumHeight(maxheight)
font = self.logOutput.font()
font.setFamily("Courier")

Write Preview
Loading…
Cancel
Save