Code tidy to ensure it complies with the Python coding standard (10/10 from PyLint).
This commit is contained in:
parent
0a519009e3
commit
e5df7f77f7
89
blocklist.py
89
blocklist.py
|
@ -4,8 +4,9 @@
|
||||||
import sys
|
import sys
|
||||||
import logging
|
import logging
|
||||||
|
|
||||||
class BlockList:
|
class BlockList(object):
|
||||||
"""Class to perform basic reading of Sentor blocklist URLs and add them to a common data store."""
|
"""Class to perform basic reading of Sentor blocklist URLs and add them
|
||||||
|
to a common data store."""
|
||||||
data = {}
|
data = {}
|
||||||
|
|
||||||
def __init__(self):
|
def __init__(self):
|
||||||
|
@ -14,53 +15,80 @@ class BlockList:
|
||||||
import os
|
import os
|
||||||
|
|
||||||
# Find the configuration file in the same directory at the main script.
|
# Find the configuration file in the same directory at the main script.
|
||||||
config_file = os.path.join(os.path.dirname(sys.argv[0]), "blocklist.cfg")
|
config_file = os.path.join(
|
||||||
|
os.path.dirname(sys.argv[0]),
|
||||||
|
"blocklist.cfg"
|
||||||
|
)
|
||||||
try:
|
try:
|
||||||
self.config = ConfigParser.ConfigParser()
|
self.config = ConfigParser.ConfigParser()
|
||||||
self.config.readfp(open(config_file))
|
self.config.readfp(open(config_file))
|
||||||
except (IOError, ConfigParser.MissingSectionHeaderError), error:
|
except (IOError, ConfigParser.MissingSectionHeaderError), error:
|
||||||
logging.error("Could not read configuration file %s: %s", config_file, error)
|
logging.error(
|
||||||
|
"Could not read configuration file %s: %s",
|
||||||
|
config_file,
|
||||||
|
error
|
||||||
|
)
|
||||||
raise
|
raise
|
||||||
|
|
||||||
def read(self, source):
|
def read(self, source):
|
||||||
"""Parse the blocklist from the provided url (source) using a CSV parser."""
|
"""Parse the blocklist from the provided url (source) using a
|
||||||
|
CSV parser."""
|
||||||
import csv
|
import csv
|
||||||
|
|
||||||
try:
|
try:
|
||||||
# Parse the Sentor Assassin blocklist format (easist to use a CSV parser)
|
# Parse the Sentor Assassin blocklist format
|
||||||
reader = csv.reader(self.cache(source))
|
# (easist to use a CSV parser)
|
||||||
|
reader = csv.reader(self.cache(source))
|
||||||
for line in reader:
|
for line in reader:
|
||||||
# Fetch the items from the input
|
# Fetch the items from the input
|
||||||
(remote_ip, forwarded_ip, useragent, cookie) = line
|
(remote_ip, forwarded_ip, useragent, cookie) = line
|
||||||
self.add(remote_ip, forwarded_ip, useragent, cookie)
|
self.add(remote_ip, forwarded_ip, useragent, cookie)
|
||||||
except csv.Error, error:
|
except csv.Error, error:
|
||||||
logging.error("There was an error retrieving the blocklist. %s", error)
|
logging.error(
|
||||||
|
"There was an error retrieving the blocklist. %s",
|
||||||
|
error
|
||||||
|
)
|
||||||
|
|
||||||
def add(self, remote_ip, forwarded_ip, useragent, cookie):
|
def add(self, remote_ip, forwarded_ip, useragent, cookie):
|
||||||
"""Method to store the remote_ip, forwarded_ip, useragent and cookie to the in-memory dictionary."""
|
"""Method to store the remote_ip, forwarded_ip, useragent and cookie
|
||||||
|
to the in-memory dictionary."""
|
||||||
# Store the various items
|
# Store the various items
|
||||||
if remote_ip not in self.data:
|
if remote_ip not in self.data:
|
||||||
self.data[remote_ip] = { "remote_ip": remote_ip, "forwarded_ip": forwarded_ip, "useragent": useragent, "cookie": cookie }
|
self.data[remote_ip] = {
|
||||||
|
"remote_ip": remote_ip,
|
||||||
|
"forwarded_ip": forwarded_ip,
|
||||||
|
"useragent": useragent,
|
||||||
|
"cookie": cookie
|
||||||
|
}
|
||||||
else:
|
else:
|
||||||
logging.debug("%s already exists in blacklist. Ignoring.", remote_ip)
|
logging.debug(
|
||||||
|
"%s already exists in blacklist. Ignoring.",
|
||||||
|
remote_ip
|
||||||
|
)
|
||||||
|
|
||||||
def cache(self, source):
|
def cache(self, source):
|
||||||
"""Attempt to read from the source URL and store results in a cache file, otherwise use the contents of the cache. If the cache isn't usable but the data is still available, return the transient data."""
|
"""Attempt to read from the source URL and store results in a cache
|
||||||
|
file, otherwise use the contents of the cache. If the cache isn't
|
||||||
|
usable but the data is still available, return the transient data."""
|
||||||
import urllib2
|
import urllib2
|
||||||
import urlparse
|
import urlparse
|
||||||
import os
|
import os
|
||||||
|
|
||||||
# Build some 'handy' variables
|
# Build some 'handy' variables
|
||||||
hostname = urlparse.urlparse(source)[1]
|
hostname = urlparse.urlparse(source)[1]
|
||||||
cache_dir = self.config.get("cache", "directory")
|
cache_dir = self.config.get("cache", "directory")
|
||||||
cache_path = os.path.join(cache_dir, "%s.cache" % hostname )
|
cache_path = os.path.join(cache_dir, "%s.cache" % hostname)
|
||||||
|
|
||||||
# Create the caching directory
|
# Create the caching directory
|
||||||
if not os.path.exists(cache_dir):
|
if not os.path.exists(cache_dir):
|
||||||
try:
|
try:
|
||||||
os.makedirs(cache_dir)
|
os.makedirs(cache_dir)
|
||||||
except OSError, error:
|
except OSError, error:
|
||||||
logging.warning("Could not create the caching directory: %s Will attempt to run without a cache.", error)
|
logging.warning(
|
||||||
|
"Could not create the caching directory: %s." +
|
||||||
|
"Will attempt to run without a cache.",
|
||||||
|
error
|
||||||
|
)
|
||||||
|
|
||||||
# Attempt to fetch the data and store it in a cache file
|
# Attempt to fetch the data and store it in a cache file
|
||||||
try:
|
try:
|
||||||
|
@ -70,16 +98,30 @@ class BlockList:
|
||||||
cache_file.write(raw_data)
|
cache_file.write(raw_data)
|
||||||
except (urllib2.URLError, urllib2.HTTPError), error:
|
except (urllib2.URLError, urllib2.HTTPError), error:
|
||||||
# Network error. Warn and use the cached content.
|
# Network error. Warn and use the cached content.
|
||||||
logging.warning("Reverting to cache file. There was a problem contacting host %s: %s", hostname, error)
|
logging.warning(
|
||||||
|
"Reverting to cache file. There was a problem contacting" +
|
||||||
|
"host %s: %s",
|
||||||
|
hostname,
|
||||||
|
error
|
||||||
|
)
|
||||||
try:
|
try:
|
||||||
cache_file = open(cache_path, "r")
|
cache_file = open(cache_path, "r")
|
||||||
except IOError, error:
|
except IOError, error:
|
||||||
logging.error("No cache file was available for %s: %s", hostname, error)
|
logging.error(
|
||||||
|
"No cache file was available for %s: %s",
|
||||||
|
hostname,
|
||||||
|
error
|
||||||
|
)
|
||||||
raise
|
raise
|
||||||
except IOError, error:
|
except IOError, error:
|
||||||
# Cache error, but network succeeded. Use String IO to return the data.
|
# Cache error, but network succeeded. Use String IO
|
||||||
|
# to return the data.
|
||||||
import StringIO
|
import StringIO
|
||||||
logging.warning("Could not create cache file: %s. Returning transient data.", error)
|
logging.warning(
|
||||||
|
"Could not create cache file: %s." +
|
||||||
|
"Returning transient data.",
|
||||||
|
error
|
||||||
|
)
|
||||||
cache_file = StringIO.StringIO()
|
cache_file = StringIO.StringIO()
|
||||||
cache_file.write(raw_data)
|
cache_file.write(raw_data)
|
||||||
|
|
||||||
|
@ -98,4 +140,9 @@ class BlockList:
|
||||||
def export(self):
|
def export(self):
|
||||||
"""Output a plaintext blocklist to stdout."""
|
"""Output a plaintext blocklist to stdout."""
|
||||||
for item in self.data.values():
|
for item in self.data.values():
|
||||||
print "%s,%s,%s,%s" % (item["remote_ip"], item["forwarded_ip"], item["useragent"], item["cookie"])
|
print "%s,%s,%s,%s" % (
|
||||||
|
item["remote_ip"],
|
||||||
|
item["forwarded_ip"],
|
||||||
|
item["useragent"],
|
||||||
|
item["cookie"]
|
||||||
|
)
|
||||||
|
|
Loading…
Reference in a new issue