Commit 8f024c1b authored by Markus Frei's avatar Markus Frei
Browse files

Merge branch 'develop' into 'master'

new release

See merge request linuxfabrik-icinga-plugins/lib-linux!3
parents 39e0984c 6dda2060
# needed in order to be able to import from this directory
\ No newline at end of file
# needed in order to be able to import from this directory
......@@ -8,32 +8,49 @@
# https://git.linuxfabrik.ch/linuxfabrik-icinga-plugins/checks-linux/-/blob/master/CONTRIBUTING.md
__author__ = 'Linuxfabrik GmbH, Zurich/Switzerland'
__version__ = '2020040701'
"""Extends argparse by new input argument data types on demand.
"""
__author__ = 'Linuxfabrik GmbH, Zurich/Switzerland'
__version__ = '2020043001'
def csv(string):
return [x.strip() for x in string.split(',')]
def csv(arg):
"""Returns a list from a `csv` input argument.
"""
def float_or_none(input):
if input is None or str(input.lower()) == 'none':
return [x.strip() for x in arg.split(',')]
def float_or_none(arg):
"""Returns None or float from a `float_or_none` input argument.
"""
if arg is None or str(arg.lower()) == 'none':
return None
return float(input)
return float(arg)
def int_or_none(arg):
"""Returns None or int from a `int_or_none` input argument.
"""
def int_or_none(input):
if input is None or str(input.lower()) == 'none':
if arg is None or str(arg.lower()) == 'none':
return None
return int(input)
return int(arg)
def range_or_none(input):
return str_or_none(input)
def range_or_none(arg):
"""Returns None or range from a `range_or_none` input argument.
"""
return str_or_none(arg)
def str_or_none(input):
if input is None or str(input.lower()) == 'none':
return None
return str(input)
def str_or_none(arg):
"""Returns None or str from a `str_or_none` input argument.
"""
if arg is None or str(arg.lower()) == 'none':
return None
return str(arg)
This diff is collapsed.
......@@ -15,7 +15,7 @@ simply return `False`.
>>> cache.get('session-key')
False
>>> cache.set('session-key', '123abc', expire=int(time.time()) + 5)
>>> cache.set('session-key', '123abc', expire=base.now() + 5)
True
>>> cache.get('session-key')
u'123abc'
......@@ -24,14 +24,14 @@ u'123abc'
False
"""
__author__ = 'Linuxfabrik GmbH, Zurich/Switzerland'
__version__ = '2020040701'
__author__ = 'Linuxfabrik GmbH, Zurich/Switzerland'
__version__ = '2020051301'
import base
import db_sqlite
def get(key):
def get(key, as_dict=False):
"""Get the value of key. If the key does not exist, `False` is returned.
Parameters
......@@ -50,21 +50,41 @@ def get(key):
if not success:
return False
success, result = db_sqlite.select(conn,
sql='SELECT key, value, timestamp FROM cache WHERE key = :key',
success, result = db_sqlite.select(
conn,
sql='SELECT key, value, timestamp FROM cache WHERE key = :key;',
data={'key': key}, fetchone=True
)
db_sqlite.close(conn)
if not success:
# error accessing or querying the cache
db_sqlite.close(conn)
return False
if not result or result == None or (result['timestamp'] != 0 and result['timestamp'] - base.now() < 0):
if not result or result is None:
# key not found
db_sqlite.close(conn)
return False
else:
# return the value
return result['value']
return False
if result['timestamp'] != 0 and result['timestamp'] <= base.now():
# key was found, but timstamp was set and has expired:
# delete all expired keys and return false
data = {'key' : result['key']}
success, result = db_sqlite.delete(
conn,
sql='DELETE FROM cache WHERE timestamp <= {};'.format(base.now())
)
success, result = db_sqlite.commit(conn)
db_sqlite.close(conn)
return False
# return the value
db_sqlite.close(conn)
if not as_dict:
# just return the value (as used to when for example using Redis)
return result['value']
# return all columns
return result
def set(key, value, expire=0):
......@@ -72,7 +92,7 @@ def set(key, value, expire=0):
Keys have to be unique. If the key already holds a value, it is
overwritten, including the expire timestamp in seconds.
Parameters
----------
key : str
......@@ -113,11 +133,12 @@ def set(key, value, expire=0):
'value': value,
'timestamp': expire,
}
success, result = db_sqlite.replace(conn, data, table='cache')
if not success:
db_sqlite.close(conn)
return False
success, result = db_sqlite.commit(conn)
db_sqlite.close(conn)
if not success:
......
#! /usr/bin/env python2
# -*- encoding: utf-8; py-indent-offset: 4 -*-
#
# Author: Linuxfabrik GmbH, Zurich, Switzerland
# Contact: info (at) linuxfabrik (dot) ch
# https://www.linuxfabrik.ch/
# License: The Unlicense, see LICENSE file.
# https://git.linuxfabrik.ch/linuxfabrik-icinga-plugins/checks-linux/-/blob/master/CONTRIBUTING.md
"""Library for accessing MySQL/MariaDB servers.
For details have a look at
https://dev.mysql.com/doc/connector-python/en/
"""
__author__ = 'Linuxfabrik GmbH, Zurich/Switzerland'
__version__ = '2020050101'
try:
import mysql.connector
except ImportError as e:
print('Python module "mysql.connector" is not installed.')
exit(3)
import base
import disk
if base.version(mysql.connector.__version__) < base.version('2.0.0'):
try:
import MySQLdb.cursors
except ImportError as e:
print('Python module "MySQLdb.cursors" is not installed.')
exit(3)
def close(conn):
"""This closes the database connection.
"""
try:
conn.close()
except:
pass
return True
def commit(conn):
"""Save (commit) any changes.
"""
try:
conn.commit()
except Exception as e:
return(False, 'Error: {}'.format(e))
return (True, None)
def connect(mysql_connection):
"""Connect to a MySQL/MariaDB. `mysql_connection` has to be a dict.
>>> mysql_connection = {
... 'user': args.USERNAME,
... 'password': args.PASSWORD,
... 'host': args.HOSTNAME,
... 'database': args.DATABASE,
... 'raise_on_warnings': True
... }
>>> conn = connect(mysql_connection)
"""
try:
conn = mysql.connector.connect(**mysql_connection)
except Exception as e:
return(False, 'Connecting to DB failed, Error: {}'.format(e))
return (True, conn)
def select(conn, sql, data={}, fetchone=False):
"""The SELECT statement is used to query the database. The result of a
SELECT is zero or more rows of data where each row has a fixed number
of columns. A SELECT statement does not make any changes to the
database.
"""
if base.version(mysql.connector.__version__) >= base.version('2.0.0'):
cursor = conn.cursor(dictionary=True)
else:
cursor = conn.cursor(MySQLdb.cursors.DictCursor)
try:
if data:
cursor.execute(sql, data)
else:
cursor.execute(sql)
if fetchone:
return (True, [cursor.fetchone()])
return (True, cursor.fetchall())
except Exception as e:
return(False, 'Query failed: {}, Error: {}, Data: {}'.format(sql, e, data))
......@@ -8,35 +8,39 @@
# https://git.linuxfabrik.ch/linuxfabrik-icinga-plugins/checks-linux/-/blob/master/CONTRIBUTING.md
"""This is one typical use case of this library (from `disk-io`):
"""This is one typical use case of this library (taken from `disk-io`):
>>> conn = lib.base.coe(lib.db_sqlite.connect(filename='disk-io.db'))
>>> lib.base.coe(lib.db_sqlite.create_table(conn, definition, drop_table_first=False))
>>> lib.base.coe(lib.db_sqlite.create_index(conn, 'name')) # optional
>>> conn = lib.base.coe(lib.db_sqlite.connect(filename='disk-io.db'))
>>> lib.base.coe(lib.db_sqlite.create_table(conn, definition, drop_table_first=False))
>>> lib.base.coe(lib.db_sqlite.create_index(conn, 'name')) # optional
>>> lib.base.coe(lib.db_sqlite.insert(conn, data))
>>> lib.base.coe(lib.db_sqlite.cut(conn, max=args.COUNT*len(disks)))
>>> lib.base.coe(lib.db_sqlite.commit(conn))
>>> lib.base.coe(lib.db_sqlite.insert(conn, data))
>>> lib.base.coe(lib.db_sqlite.cut(conn, max=args.COUNT*len(disks)))
>>> lib.base.coe(lib.db_sqlite.commit(conn))
>>> result = lib.base.coe(lib.db_sqlite.select(conn,
'SELECT * FROM perfdata WHERE name = :name ORDER BY timestamp DESC LIMIT 2',
{'name': disk}
>>> lib.db_sqlite.close(conn)
"""
>>> result = lib.base.coe(lib.db_sqlite.select(conn,
'SELECT * FROM perfdata WHERE name = :name ORDER BY timestamp DESC LIMIT 2',
{'name': disk}
__author__ = 'Linuxfabrik GmbH, Zurich/Switzerland'
__version__ = '2020041002'
>>> lib.db_sqlite.close(conn)
"""
import disk
__author__ = 'Linuxfabrik GmbH, Zurich/Switzerland'
__version__ = '2020051701'
import os
import sqlite3
import base
import disk
def close(conn):
# We can close the connection if we are done with it.
# Just be sure any changes have been committed or they will be lost.
"""This closes the database connection. Note that this does not
automatically call commit(). If you just close your database connection
without calling commit() first, your changes will be lost.
"""
try:
conn.close()
except:
......@@ -45,15 +49,21 @@ def close(conn):
def commit(conn):
# Save (commit) any changes
"""Save (commit) any changes.
"""
try:
conn.commit()
except Exception as e:
return(False, 'Committing to DB failed, Error: '.format(e))
return(False, 'Error: {}'.format(e))
return (True, None)
def connect(path='', filename=''):
"""Connect to a SQLite database file. If path is ommitted, the
temporary directory is used. If filename is ommitted,
`linuxfabrik-plugins.db` is used.
"""
def get_filename(path='', filename=''):
"""Returns a path including filename to a sqlite database file.
......@@ -84,18 +94,26 @@ def connect(path='', filename=''):
# https://stackoverflow.com/questions/3300464/how-can-i-get-dict-from-sqlite-query
conn.row_factory = sqlite3.Row
except Exception as e:
return(False, 'Connecting to DB {} failed, Error: '.format(db, e))
return(False, 'Connecting to DB {} failed, Error: {}'.format(db, e))
return (True, conn)
def create_index(conn, column_list, table='perfdata', unique=False):
index_name = 'idx_{}_{}'.format(table, column_list.replace(',', '_').replace(' ', ''))
"""Creates one index on a list of/one database column/s.
"""
table = base.filter_str(table)
index_name = 'idx_{}'.format(base.md5sum(table + column_list))
c = conn.cursor()
if unique:
sql = 'CREATE UNIQUE INDEX IF NOT EXISTS {} ON "{}" ({});'.format(index_name, table, column_list)
sql = 'CREATE UNIQUE INDEX IF NOT EXISTS {} ON "{}" ({});'.format(
index_name, table, column_list
)
else:
sql = 'CREATE INDEX IF NOT EXISTS {} ON "{}" ({});'.format(index_name, table, column_list)
sql = 'CREATE INDEX IF NOT EXISTS {} ON "{}" ({});'.format(
index_name, table, column_list
)
try:
c.execute(sql)
except Exception as e:
......@@ -104,9 +122,15 @@ def create_index(conn, column_list, table='perfdata', unique=False):
return (True, True)
# create_table('test', 'a,b,c') results in
# CREATE TABLE "test" (a TEXT, b TEXT, c TEXT)
def create_table(conn, definition, table='perfdata', drop_table_first=False):
"""Create a database table.
>>> create_table('test', 'a,b,c INTEGER NOT NULL')
results in 'CREATE TABLE "test" (a TEXT, b TEXT, c INTEGER NOT NULL)'
"""
table = base.filter_str(table)
# create table if it does not exist
if drop_table_first:
success, result = drop_table(conn, table)
......@@ -124,11 +148,15 @@ def create_table(conn, definition, table='perfdata', drop_table_first=False):
def cut(conn, table='perfdata', max=5):
# keep only the latest "max" records, using the sqlite built-in "rowid"
"""Keep only the latest "max" records, using the sqlite built-in "rowid".
"""
table = base.filter_str(table)
c = conn.cursor()
sql = '''DELETE FROM {} WHERE rowid IN (
SELECT rowid FROM {} ORDER BY rowid DESC LIMIT -1 OFFSET :max
)'''.format(table, table)
sql = '''DELETE FROM {table} WHERE rowid IN (
SELECT rowid FROM {table} ORDER BY rowid DESC LIMIT -1 OFFSET :max
);'''.format(table=table)
try:
c.execute(sql, (max, ))
except Exception as e:
......@@ -137,7 +165,35 @@ def cut(conn, table='perfdata', max=5):
return (True, True)
def delete(conn, sql, data={}, fetchone=False):
"""The DELETE command removes records from a table. If the WHERE
clause is not present, all records in the table are deleted. If a
WHERE clause is supplied, then only those rows for which the WHERE
clause boolean expression is true are deleted. Rows for which the
expression is false or NULL are retained.
"""
c = conn.cursor()
try:
if data:
return (True, c.execute(sql, data).rowcount)
else:
return (True, c.execute(sql).rowcount)
except Exception as e:
return(False, 'Query failed: {}, Error: {}, Data: {}'.format(sql, e, data))
def drop_table(conn, table='perfdata'):
"""The DROP TABLE statement removes a table added with the CREATE TABLE
statement. The name specified is the table name. The dropped table is
completely removed from the database schema and the disk file. The
table can not be recovered. All indices and triggers associated with the
table are also deleted.
"""
table = base.filter_str(table)
c = conn.cursor()
sql = 'DROP TABLE IF EXISTS "{}";'.format(table)
......@@ -150,7 +206,11 @@ def drop_table(conn, table='perfdata'):
def insert(conn, data, table='perfdata'):
# insert a row of data (values from a dictionary)
"""Insert a row of values (= dict).
"""
table = base.filter_str(table)
c = conn.cursor()
sql = 'INSERT INTO "{}" (COLS) VALUES (VALS);'.format(table)
......@@ -171,6 +231,20 @@ def insert(conn, data, table='perfdata'):
def replace(conn, data, table='perfdata'):
"""The REPLACE command is an alias for the "INSERT OR REPLACE" variant
of the INSERT command. When a UNIQUE or PRIMARY KEY constraint violation
occurs, it does the following:
* First, delete the existing row that causes a constraint violation.
* Second, insert a new row.
In the second step, if any constraint violation e.g., NOT NULL
constraint occurs, the REPLACE statement will abort the action and roll
back the transaction.
"""
table = base.filter_str(table)
c = conn.cursor()
sql = 'REPLACE INTO "{}" (COLS) VALUES (VALS);'.format(table)
......@@ -190,7 +264,13 @@ def replace(conn, data, table='perfdata'):
return (True, True)
def select(conn, sql, data={}, table='perfdata', fetchone=False):
def select(conn, sql, data={}, fetchone=False, as_dict=True):
"""The SELECT statement is used to query the database. The result of a
SELECT is zero or more rows of data where each row has a fixed number
of columns. A SELECT statement does not make any changes to the
database.
"""
c = conn.cursor()
try:
......@@ -199,9 +279,96 @@ def select(conn, sql, data={}, table='perfdata', fetchone=False):
else:
c.execute(sql)
# https://stackoverflow.com/questions/3300464/how-can-i-get-dict-from-sqlite-query
if fetchone:
return (True, [dict(row) for row in c.fetchall()][0])
else:
if as_dict:
if fetchone:
return (True, [dict(row) for row in c.fetchall()][0])
return (True, [dict(row) for row in c.fetchall()])
if fetchone:
return (True, c.fetchone())
return (True, c.fetchall())
except Exception as e:
return(False, 'Query failed: {}, Error: {}, Data: {}'.format(sql, e, data))
def get_tables(conn):
"""List all tables in a database.
"""
sql = "SELECT name FROM sqlite_master WHERE type ='table' AND name NOT LIKE 'sqlite_%';"
return select(conn, sql)
def compute_load(conn, sensorcol, datacols, count, table='perfdata'):
"""Calculates Load1 and Loadn (n = count). Load is caclulated as a
"per second" number.
The Perfdata table must have a "timestamp" column.
>>> compute_load(conn, sensorcol='interface', datacols=['tx_bytes',
'rx_bytes'], count=5, table='perfdata')
Returns
-------
list
[{'interface': u'mgmt1', 'tx_bytes1': 6906, 'rx_bytes1': 10418,
'rx_bytesn': 10871, 'tx_bytesn': 7442},
{...},
]
"""
table = base.filter_str(table)
# count the number of different sensors in the perfdata table
sql = 'SELECT DISTINCT {sensorcol} FROM {table} ORDER BY {sensorcol} ASC;'.format(
sensorcol=sensorcol, table=table
)
success, sensors = select(conn, sql)
if not success:
return (False, sensors)
if len(sensors) == 0:
return (True, False)
load = []
# calculate
for sensor in sensors:
# get all historical data, ordered by sensor, and within that newest data first
sensor_name = sensor[sensorcol]
success, perfdata = select(
conn,
'SELECT * FROM {table} WHERE {sensorcol} = :{sensorcol} '
'ORDER BY timestamp DESC;'.format(
table=table, sensorcol=sensorcol
),
data={sensorcol: sensor_name})
if not success:
return (False, perfdata)
# not enough data to compute load
if len(perfdata) < count:
return (True, False)
# Perfdata:
# [{'interface': u'mgmt1', 'tx_bytes': 102893695, 'timestamp': 1588162358}, ...
# perfdata[0]: newest/current entry
# perfdata[1]: one entry before, load1 = ([0] - [1])/seconds
# perfdata[count-1]: oldest entry, loadn = ([0] - [n])/seconds
load1_delta = perfdata[0]['timestamp'] - perfdata[1]['timestamp']
loadn_delta = perfdata[0]['timestamp'] - perfdata[count-1]['timestamp']
tmp = {}
tmp[sensorcol] = sensor_name
for key in datacols:
if key in perfdata[0]:
if load1_delta != 0:
tmp[key + '1'] = (perfdata[0][key] - perfdata[1][key]) / load1_delta
else:
tmp[key + '1'] = 0
if loadn_delta != 0:
tmp[key + 'n'] = (perfdata[0][key] - perfdata[count-1][key]) / loadn_delta
else:
tmp[key + 'n'] = 0
load.append(tmp)
return (True, load)
......@@ -8,29 +8,48 @@
# https://git.linuxfabrik.ch/linuxfabrik-icinga-plugins/checks-linux/-/blob/master/CONTRIBUTING.md
__author__ = 'Linuxfabrik GmbH, Zurich/Switzerland'
__version__ = '2020040901'
"""Offers file and disk related functions, like getting a list of
partitions, grepping a file, etc.
"""
from lib.globals import *
__author__ = 'Linuxfabrik GmbH, Zurich/Switzerland'
__version__ = '2020051001'
import os
import re
import sys
import tempfile
from lib.globals import STATE_UNKNOWN
try:
import psutil
except ImportError, e:
except ImportError as e:
print('Python module "psutil" is not installed.')
exit(STATE_UNKNOWN)
import re
import tempfile
sys.exit(STATE_UNKNOWN)
def get_cwd():
"""Gets the current working directory.
"""