rawlog data in a lua script
João Silva
joaopfmlist at lipc.fis.uc.pt
Thu Jul 28 13:38:17 UTC 2022
Thanks a lot for this thread.
I was starting to plan a system where multiples processes can write to a
file and completely forgot that syslog is designed to do that.
It is a perfect solution, I only had to configure a local facility to
receive the data and add 3 lines of code to the program (including the
import).
On 28/07/2022 12:01, Paul Kudla (SCOM.CA Internet Services Inc.) wrote:
>
> Hi - I use this python script to capture a socket (ie the log file)
> and then send it to syslog, i use this for all the systems that do not
> really support syslogging (apache etc)
>
> basic useage
>
> /usr/bin/nohup /programs/common/capture -s
> /usr/local/apache2/logs/httpd-access.log -l httpd -d 10.228.0.6:514 -p
> httpd & > /dev/null
>
> i typically run this at startup in rc.local
>
> hope this helps :
>
> ----------------------------------------------------------------------
>
> ## cat capture
> #!/usr/local/bin/python3
> # -*- coding: UTF-8 -*-
>
>
> import os,sys,socket
> import datetime,time
> from optparse import OptionParser
>
> from lib import *
>
> USAGE_TEXT = '''\
> usage: %%prog %s[options]
> '''
>
> parser = OptionParser(usage=USAGE_TEXT % '', version='0.4')
>
> parser.add_option("-s", "--socket", dest="socket_file", help="Socket
> File to Capture")
> parser.add_option("-l", "--label", dest="label", help="Syslog Label to
> Insert")
> parser.add_option("-d", "--destination", dest="destination",
> help="Syslog Destibnation Server:Port")
> parser.add_option("-p", "--pid", dest="pid", help="PID Process Name")
> #parser.add_option("-e", "--email", dest="email", help="Additional
> Email To")
> #parser.add_option("-t", "--temp", dest="tempdir", help="Local Temp
> Directory")
>
> options, args = parser.parse_args()
>
> print (options.socket_file)
> print (options.label)
> print (options.destination)
> print (options.pid)
>
>
>
> if options.socket_file == None :
> print ('Missing Socket File Information')
> sys.exit()
>
> if options.label == None :
> print ('Missing Syslog Label Information')
> sys.exit()
>
> if options.destination == None :
> print ('Missing Syslog Destination host:[port]')
> sys.exit()
>
> if options.pid == None :
> print ('Missing Syslog Pid Process Name')
> sys.exit()
>
>
> #try local syslog (/var/run/log)
>
> UDP_IP = options.destination.split(':')
>
> if len(UDP_IP) == 2 : #Set Port
> UDP_PORT = int(UDP_IP[1])
> else :
> UDP_PORT = 514 #Default
>
> UDP_IP = UDP_IP[0] #Server
>
> #MESSAGE = str("<22>Mar 27 04:16:16 es-scom[12345] offsite.scom.ca su:
> Hello, World!")
> #MESSAGE = str("<183>Mar 27 16:17:41 scom-live[72178]: Hello World")
>
> print("UDP target IP: %s" % UDP_IP)
> print("UDP target port: %s" % UDP_PORT)
> #print("message: %s" % MESSAGE)
>
> count = 10
>
>
> #sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
> #sock.sendto(MESSAGE, (UDP_IP, UDP_PORT))
> #sock.sendto(bytes(MESSAGE, "utf-8"), (UDP_IP, UDP_PORT))
> #sock.close()
> #sys.exit()
>
>
> #def read_commands():
> try:
> print ("Creating read pipe... %s" %options.socket_file )
> os.mkfifo(options.socket_file) # Create pipe
> print ("Pipe %s created!" %options.socket_file )
> except:
> print ("Pipe %s already exists" %options.socket_file )
>
> #chmod 777 the file so everyone can talk to it
> os.system('/bin/chmod 777 %s' %options.socket_file)
>
>
> with open(options.socket_file, "r") as pipecmd:
> while True:
> time.sleep(.001)
> try:
> line = pipecmd.readline()
> if line != '' : #New Data
> if line == '\n' :
> continue
> print ('Raw Text : %s' %line)
> encoded_string = line.encode("ascii",
> "ignore")
> line = encoded_string.decode()
> line = create_ascii(line)
> line = line.ascii
> print ('Line after ASCII : %s' %line)
> print ( 'Line Count : %s' %len(line) )
> #line = data
> #go get my pid
> pid_process = '0'
> if options.pid == 'postfix' : #its a
> diverted postfix process get the actual pid from raw text
> pid_process =
> line.split('[',1)[1].split(']',1)[0]
>
> else :
> command = commands('/bin/ps
> -axww | /usr/bin/grep %s' %options.pid)
> print ()
> #print (command.output)
>
> for n in range
> (0,len(command.output)) :
> if '/bin/ps -axww |
> /usr/bin/grep' not in command.output[n] and '/usr/bin/grep' not in
> command.output[n] and '/usr/local/bin/python3' not in
> command.output[n] :
> pid_process = ( command.output.split(' ')[0] ) #whats left should be
> my process ?
> break
>
> print ('PID Process : %s ' %pid_process )
>
> if options.destination == 'local' :
> #Send to log here
> print ('Sending to Local Syslog')
> log = open ('/var/run/log','w')
> log.write ('hello')
> log.close()
> sys.exit()
>
>
> else : #Send via socket
> #Make the line in freebsd
> syslog format
> MESSAGE = '<' + str(count) +
> '>' + str( time.strftime("%b %d %H:%M:%S ") ) + str(options.label) +
> '[' + str(pid_process) + ']: ' + str(line)
> print ('Sent : %s' %MESSAGE )
> count = count + 1
> if count > 255 :
> count = 10
>
> # send to udp logger port
> specified
> sock =
> socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
> sock.sendto(bytes(MESSAGE,
> "utf-8"), (UDP_IP, UDP_PORT))
> sock.close()
>
>
> else : #No data
> pass
>
> except Exception as e:
> exc_type, exc_obj, exc_tb = sys.exc_info()
> fname =
> os.path.split(exc_tb.tb_frame.f_code.co_filename)[1]
> e = str(e) + '\n\n' + str(exc_type) + '\n' +
> str(fname) + '\n' + str(exc_tb.tb_lineno)
>
> print ('\n\nCaught Exception : %s' %e )
>
> print ("Could not read cmd pipe, skipping ...")
>
>
> sys.exit()
>
> -----------------------------------------------------------------------
>
>
> and lib.py
>
>
>
> ----------------------------------------------------------------------
>
>
> ## cat lib3.py
> #Python Library written by paul kudla (c) 2011
>
> #Load the librarys for the system
>
> import os,sys,time,socket
> import string
> from ftplib import FTP
> from decimal import *
> from datetime import date
> import datetime
> import smtplib
> from email.mime.multipart import MIMEMultipart
> from email.mime.base import MIMEBase
> from email.mime.text import MIMEText
> from email.utils import COMMASPACE, formatdate
> from email import encoders
> import subprocess
>
> getcontext().prec = 20
>
>
> class commands:
> def __init__(self,command) :
> self.command = command
> #print (self.command)
> self.output = 'Error'
> self.status = '255'
>
> #sample
> #rc, gopath = subprocess.getstatusoutput('ls -a')
>
> self.status, self.output =
> subprocess.getstatusoutput(self.command)
>
> try:
> self.cr = self.output.split('\n')
> except :
> self.cr = []
> try:
> self.count = len(self.cr)
> except :
> self.count = 0
>
> self.status = int(self.status)
>
> #return count=number of lines, cr = lines split,
> getoutput = actual output returned, status = return code
>
> return
>
> #Email with attachment
> class sendmail:
> def __init__(self, send_from, send_to, send_subject,
> send_text, send_files):
> #send_from, send_to, send_subject, send_text, send_files):
> #print ('lib.py sending email')
> assert type(send_to)==list
> assert type(send_files)==list
>
> msg = MIMEMultipart()
> msg['From'] = send_from
> msg['To'] = COMMASPACE.join(send_to)
> msg['Date'] = formatdate(localtime=True)
> msg['Subject'] = send_subject
>
> msg.attach( MIMEText(send_text) )
>
> for f in send_files:
> part = MIMEBase('application', "octet-stream")
> part.set_payload( open(f,"rb").read() )
> Encoders.encode_base64(part)
> part.add_header('Content-Disposition',
> 'attachment; filename="%s"' % os.path.basename(f))
> msg.attach(part)
>
> try : #Send Local?
> smtp = smtplib.SMTP('mail.local.scom.ca')
> #smtp.login('backup at scom.ca','522577')
> #print ('Sending Email to : %s' %send_to)
> smtp.sendmail(send_from, send_to,
> msg.as_string())
> smtp.close()
>
> except :
> smtp = smtplib.SMTP('mail.scom.ca')
> smtp.login('backup at scom.ca','522577')
> #print ('Sending Email to : %s' %send_to)
> smtp.sendmail(send_from, send_to,
> msg.as_string())
> smtp.close()
>
> class getdatetime:
> def __init__(self):
> self.datetime = datetime.date.today()
> self.datetime_now = datetime.datetime.now()
> self.date = str( time.strftime("%Y-%m-%d %H:%M:%S") )
> self.date_long = str( time.strftime("%Y-%m-%d
> %H:%M:%S") )
> self.date_short = str( time.strftime("%Y-%m-%d") )
> self.time = str( time.strftime("%H:%M:%S") )
> self.date_time_sec = self.datetime_now.strftime
> ("%Y-%m-%d %H:%M:%S.%f")
>
>
> #Return edi senddate string (short) 2011-10-31 into 111031
>
> class create_ascii :
> def __init__(self,string_data) :
> self.string_data = str(string_data)
> import string
> self.printable = set(string.printable)
> self.list = list(filter(lambda x: x in self.printable,
> self.string_data))
> #print (self.list)
>
> self.ascii = ''
> for n in range (0,len(self.list)) :
> self.ascii = self.ascii + self.list[n]
> self.ascii = str(self.ascii)
>
> return
>
>
> class edi_send_date_short:
> def __init__(self, senddate):
> self.date = senddate
> self.result = self.date[2] + self.date[3] +
> self.date[5] + self.date[6] + self.date[8] + self.date[9]
>
> def __str__(self):
> return '%s' % self.result
>
> ##Return edi senddate string (long) 2011-10-31 into 20111031
> class edi_send_date_long:
> def __init__(self, senddate):
> self.date = senddate
> self.result1 = self.date[0] + self.date[1] +
> self.date[2] + self.date[3] + self.date[5] + self.date[6] +
> self.date[8] + self.date[9]
> self.result2 = self.date[2] + self.date[3] +
> self.date[5] + self.date[6] + self.date[8] + self.date[9]
>
> def __str__(self):
> return '%s' % (self.result1,self.result2)
>
> class gpsdeg:
> def __init__(self, dms):
> self.dms = dms
> self.is_positive = self.dms >= 0
> self.dms = abs(self.dms)
> self.minutes,self.seconds = divmod(self.dms*3600,60)
> self.degrees,self.minutes = divmod(self.minutes,60)
> self.degrees = self.degrees if self.is_positive else
> -self.degrees
>
> def __str__(self):
> return '%s' % (self.degrees,self.minutes,self.seconds)
>
>
> class degdir:
> def __init__(self, degrees):
> self.direction_data = ['N','348.75','11.25','NNE',
> '11.25','33.75','NE','33.75','56.25','ENE',
> '56.25','78.75','E','78.75','101.25','ESE','101.25','123.75','SE','123.75','146.25','SSE','146.25','168.75','S','168.75','191.25','SSW','191.25','213.75','SW','213.75','236.25','WSW','236.25','258.75','W','258.75','281.25','WNW','281.25','303.75','NW','303.75','326.25','NNW','326.25','348.75']
>
>
> def __str__(self):
> return '%s' % (self.direction)
>
>
> class gettime:
> def __init__(self):
> self.uu = time.localtime()
>
> self.todaystime = str(self.uu[3]) #get the hr
>
> if int(self.uu[3]) < 10: #add a zero
> self.todaystime = '0' + self.todaystime
> if int(self.uu[4]) < 10: #add a zero in front
> self.todaystime = self.todaystime
> +":0"+str(self.uu[4])
> else:
> self.todaystime = self.todaystime
> +":"+str(self.uu[4])
>
> def __str__(self):
> return self.todaystime
>
> class array2dbstring:
> def __init__(self,array):
> self.data = array
> for self.nn in range(0,len(self.data)):
> print ('Data %s \t\t %s' %
> (str(self.data[self.nn]),str( type(self.data[self.nn])) ) ) #change
> all data into strings
> self.a = type(self.data[self.nn])
> self.a = str(self.a)
> if 'Decimal' in self.a :
> self.n = str(self.data[self.nn])
> #self.n = self.n.lstrip("'")
> #self.n = self.n.rstrip("'")
> #self.data[self.nn] =
> float(self.data[self.nn])
> self.data[self.nn] = str('0.00')
> print (self.n)
>
> if 'NoneType' in self.a :
> self.data[self.nn] = ''
> if 'datetime.datetime' in self.a :
> #self.data[self.nn] =
> str(self.data[self.nn])
> #self.data[self.nn].replace
> self.data[self.nn] = '2012-01-25
> 00:00:00'
> self.data = str(self.data)
> self.data = self.data.lstrip('[')
> self.data = self.data.rstrip(']')
> self.data = self.data.replace("'NULL'","NULL")
> #self.data = self.data.replace(" '',", ",")
> #self.data = self.data.replace(" '0.00'","'100'")
>
> def __str__(self):
> return self.data
>
> class get_hostname:
> def __init__(self):
> self.hostname = socket.gethostname()
>
>
> ----------------------------------------------------------------------------
>
>
>
> Happy Thursday !!!
> Thanks - paul
>
> Paul Kudla
>
>
> Scom.ca Internet Services <http://www.scom.ca>
> 004-1009 Byron Street South
> Whitby, Ontario - Canada
> L1N 4S3
>
> Toronto 416.642.7266
> Main 1.866.411.7266
> Fax 1.888.892.7266
> Email paul at scom.ca
>
> On 7/28/2022 6:17 AM, dovecot-bounces at dovecot.org wrote:
>> Hi,
>> I'm searching for a possibility to have the rawlog feature in lua,
>> which would be much easier for processing. Currently Dovecot, when
>> activating rawlog for a user, writes everything to disk (which
>> creates I/O), and I have to somehow read it from there. That's a bit
>> complicated, because I have to get notified via inotify or similar
>> when there are new files created, and then I have to start a "tail"
>> or "epoll" mechanism on the files to get the contents in more or less
>> real time (IMAP sessions can be multiple hours or days).
>> It would be much easier to hook to the "raw request and response
>> events" inside Dovecot and have the rawlog-data in a lua script,
>> where I can prepare it and send it to another maschine for
>> monitoring/collection/analysis/statistics or similar, for example via
>> HTTP.
>> Having the rawlog data available in lua would make things a lot easier.
>> Is there any possibility at the moment to create a lua script and
>> "hook" to those "request and response events"? If not, would it be
>> possible to add that feature in the future?
>> Kind regards
>> Michael
More information about the dovecot
mailing list