"Fossies" - the Fresh Open Source Software Archive 
Member "pyzor-1.0.0/scripts/pyzor" (10 Dec 2014, 14050 Bytes) of package /linux/privat/pyzor-1.0.0.tar.gz:
As a special service "Fossies" has tried to format the requested source page into HTML format using (guessed) Python source code syntax highlighting (style:
standard) with prefixed line numbers.
Alternatively you can here
view or
download the uninterpreted source code file.
See also the latest
Fossies "Diffs" side-by-side code changes report for "pyzor":
0.9.0_vs_1.0.0.
1 #! /usr/bin/env python
2
3 """Pyzor client."""
4
5 from __future__ import print_function
6
7 import os
8 import sys
9 import email
10 import random
11 import mailbox
12 import hashlib
13 import getpass
14 import logging
15 import optparse
16 import tempfile
17 import threading
18
19 try:
20 import ConfigParser
21 except ImportError:
22 import configparser as ConfigParser
23
24 import pyzor.digest
25 import pyzor.client
26 import pyzor.config
27
28
29 def load_configuration():
30 """Load the configuration for the server.
31
32 The configuration comes from three sources: the default values, the
33 configuration file, and command-line options."""
34 # Work out the default directory for configuration files.
35 # If $HOME is defined, then use $HOME/.pyzor, otherwise use /etc/pyzor.
36 userhome = os.getenv("HOME")
37 if userhome:
38 homedir = os.path.join(userhome, '.pyzor')
39 else:
40 homedir = os.path.join("/etc", "pyzor")
41
42 # Configuration defaults. The configuration file overrides these, and
43 # then the command-line options override those.
44 defaults = {
45 "ServersFile": "servers",
46 "AccountsFile": "accounts",
47 "LocalWhitelist": "whitelist",
48 "LogFile": "",
49 "Timeout": "5", # seconds
50 "Style": "msg",
51 "ReportThreshold": "0",
52 "WhitelistThreshold": "0",
53 }
54
55 # Process any command line options.
56 description = ("Read data from stdin and execute the requested command "
57 "(one of 'check', 'report', 'ping', 'pong', 'digest', "
58 "'predigest', 'genkey', 'local_whitelist', "
59 "'local_unwhitelist').")
60 opt = optparse.OptionParser(description=description)
61 opt.add_option("-n", "--nice", dest="nice", type="int",
62 help="'nice' level", default=0)
63 opt.add_option("-d", "--debug", action="store_true", default=False,
64 dest="debug", help="enable debugging output")
65 opt.add_option("--homedir", action="store", default=homedir,
66 dest="homedir", help="configuration directory")
67 opt.add_option("-s", "--style", action="store",
68 dest="Style", default=None,
69 help="input style: 'msg' (individual RFC5321 message), "
70 "'mbox' (mbox file of messages), 'digests' (Pyzor "
71 "digests, one per line).")
72 opt.add_option("--log-file", action="store", default=None,
73 dest="LogFile", help="name of log file")
74 opt.add_option("--servers-file", action="store", default=None,
75 dest="ServersFile", help="name of servers file")
76 opt.add_option("--accounts-file", action="store", default=None,
77 dest="AccountsFile", help="name of accounts file")
78 opt.add_option("--local-whitelist", action="store", default=None,
79 dest="LocalWhitelist", help="name of the local whitelist "
80 "file")
81 opt.add_option("-t", "--timeout", dest="Timeout", type="int",
82 help="timeout (in seconds)", default=None)
83 opt.add_option("-r", "--report-threshold", dest="ReportThreshold",
84 type="int", default=None,
85 help="threshold for number of reports")
86 opt.add_option("-w", "--whitelist-threshold", dest="WhitelistThreshold",
87 type="int", default=None,
88 help="threshold for number of whitelist")
89 opt.add_option("-V", "--version", action="store_true", default=False,
90 dest="version", help="print version and exit")
91 options, args = opt.parse_args()
92
93 if options.version:
94 print("%s %s" % (sys.argv[0], pyzor.__version__))
95 sys.exit(0)
96
97 if not len(args):
98 opt.print_help()
99 sys.exit()
100 try:
101 os.nice(options.nice)
102 except AttributeError:
103 pass
104
105 # Create the configuration directory if it doesn't already exist.
106 if not os.path.exists(options.homedir):
107 os.mkdir(options.homedir)
108
109 # Load the configuration.
110 config = ConfigParser.ConfigParser()
111 # Set the defaults.
112 config.add_section("client")
113 for key, value in defaults.iteritems():
114 config.set("client", key, value)
115 # Override with the configuration.
116 config.read(os.path.join(options.homedir, "config"))
117 # Override with the command-line options.
118 for key in defaults:
119 value = getattr(options, key)
120 if value is not None:
121 config.set("client", key, str(value))
122 return config, options, args
123
124
125 def main():
126 """Execute any requested actions."""
127 # Set umask - this restricts this process from granting any world access
128 # to files/directories created by this process.
129 os.umask(0o0077)
130
131 config, options, args = load_configuration()
132
133 homefiles = ["LogFile", "ServersFile", "AccountsFile", "LocalWhitelist"]
134 pyzor.config.expand_homefiles(homefiles, "client", options.homedir, config)
135
136 logger = pyzor.config.setup_logging("pyzor",
137 config.get("client", "LogFile"),
138 options.debug)
139 servers = pyzor.config.load_servers(config.get("client", "ServersFile"))
140 accounts = pyzor.config.load_accounts(config.get("client", "AccountsFile"))
141
142 # Run the specified commands.
143 client = pyzor.client.Client(accounts,
144 int(config.get("client", "Timeout")))
145 for command in args:
146 try:
147 dispatch = DISPATCHES[command]
148 except KeyError:
149 logger.critical("Unknown command: %s", command)
150 else:
151 try:
152 if not dispatch(client, servers, config):
153 sys.exit(1)
154 except pyzor.TimeoutError:
155 # Note that most of the methods will trap their own timeout
156 # error.
157 logger.error("Timeout from server in %s", command)
158
159
160 def get_input_handler(style="msg", digester=pyzor.digest.DataDigester):
161 """Return an object that can be iterated over to get all the digests."""
162 try:
163 return INPUT_HANDLERS[style](digester)
164 except KeyError:
165 raise ValueError("Unknown input style.")
166
167
168 def _get_input_digests(dummy):
169 for line in sys.stdin:
170 yield line.strip()
171
172
173 def _get_input_msg(digester):
174 msg = email.message_from_file(sys.stdin)
175 digested = digester(msg).value
176 yield digested
177
178
179 def _get_input_mbox(digester):
180 tfile = tempfile.NamedTemporaryFile()
181 tfile.write(sys.stdin.read().encode("utf8"))
182 tfile.seek(0)
183 mbox = mailbox.mbox(tfile.name)
184 for msg in mbox:
185 digested = digester(msg).value
186 yield digested
187 tfile.close()
188
189
190 def ping(client, servers, config):
191 """Check that the server is reachable."""
192 # pylint: disable-msg=W0613
193 runner = pyzor.client.ClientRunner(client.ping)
194 for server in servers:
195 runner.run(server, (server,))
196 sys.stdout.writelines(runner.results)
197 return runner.all_ok
198
199
200 def pong(client, servers, config):
201 """Used to test pyzor."""
202 rt = int(config.get("client", "ReportThreshold"))
203 wt = int(config.get("client", "WhitelistThreshold"))
204 style = config.get("client", "Style")
205 runner = pyzor.client.CheckClientRunner(client.pong, rt, wt)
206 for digested in get_input_handler(style):
207 send_digest(digested, runner, servers)
208 sys.stdout.writelines(runner.results)
209
210 return runner.all_ok and runner.found_hit and not runner.whitelisted
211
212
213 def info(client, servers, config):
214 """Get information about each message."""
215 style = config.get("client", "Style")
216 runner = pyzor.client.InfoClientRunner(client.info)
217 for digested in get_input_handler(style):
218 send_digest(digested, runner, servers)
219 sys.stdout.writelines(runner.results)
220
221 return runner.all_ok
222
223
224 def check(client, servers, config):
225 """Check each message against each server.
226
227 The return value is 'failure' if there is a positive spam count and
228 *zero* whitelisted count; otherwise 'success'.
229 """
230 rt = int(config.get("client", "ReportThreshold"))
231 wt = int(config.get("client", "WhitelistThreshold"))
232 style = config.get("client", "Style")
233 lwhitelist_fp = config.get("client", "LocalWhitelist")
234 lwhitelist = pyzor.config.load_local_whitelist(lwhitelist_fp)
235 runner = pyzor.client.CheckClientRunner(client.check, rt, wt)
236 mock_runner = pyzor.client.CheckClientRunner(client._mock_check, rt, wt)
237 for digested in get_input_handler(style):
238 if digested in lwhitelist:
239 send_digest(digested, mock_runner, servers)
240 else:
241 send_digest(digested, runner, servers)
242 sys.stdout.writelines(mock_runner.results)
243 sys.stdout.writelines(runner.results)
244
245 return runner.all_ok and runner.found_hit and not runner.whitelisted
246
247
248 def _send_digest(runner, server, digested, spec=None):
249 """Send these digests to one server."""
250 if spec:
251 runner.run(server, (digested, server, spec))
252 else:
253 runner.run(server, (digested, server))
254
255
256 def send_digest(digested, runner, servers):
257 """Send these digests to each server."""
258 if not digested:
259 return
260
261 if len(servers) == 1:
262 _send_digest(runner, servers[0], digested)
263 return runner.all_ok
264
265 threads = []
266 for server in servers:
267 args = (runner, server, digested)
268 thread = threading.Thread(target=_send_digest, args=args)
269 threads.append(thread)
270 thread.start()
271
272 for thread in threads:
273 thread.join()
274
275 return runner.all_ok
276
277
278 def report(client, servers, config):
279 """Report each message as spam."""
280 style = config.get("client", "Style")
281 all_ok = True
282 for digested in get_input_handler(style):
283 runner = pyzor.client.ClientRunner(client.report)
284 if digested and not send_digest(digested, runner, servers):
285 all_ok = False
286 sys.stdout.writelines(runner.results)
287 return all_ok
288
289
290 def whitelist(client, servers, config):
291 """Report each message as ham."""
292 style = config.get("client", "Style")
293 all_ok = True
294 for digested in get_input_handler(style):
295 runner = pyzor.client.ClientRunner(client.whitelist)
296 if digested and not send_digest(digested, runner, servers):
297 all_ok = False
298 sys.stdout.writelines(runner.results)
299 return all_ok
300
301
302 def digest(client, servers, config):
303 """Generate a digest for each message.
304
305 This method can be used to look up digests in the database when
306 diagnosing, or to report digests in a two-stage operation (digest,
307 then report with --digests)."""
308 style = config.get("client", "Style")
309 for digested in get_input_handler(style):
310 if digested:
311 print(digested)
312 return True
313
314
315 def local_whitelist(client, servers, config):
316 """Add to the local whitelist."""
317 logger = logging.getLogger("pyzor")
318 lwhitelist_fp = config.get("client", "LocalWhitelist")
319 lwhitelist = pyzor.config.load_local_whitelist(lwhitelist_fp)
320 style = config.get("client", "Style")
321 for digested in get_input_handler(style):
322 if digested in lwhitelist:
323 logger.critical("Digest %s already whitelisted locally", digested)
324 lwhitelist.add(digested)
325 with open(lwhitelist_fp, "w") as lwhitelist_f:
326 lwhitelist_f.write("\n".join(lwhitelist))
327 return True
328
329
330 def local_unwhitelist(client, servers, config):
331 """Remove from the local whitelist."""
332 logger = logging.getLogger("pyzor")
333 lwhitelist_fp = config.get("client", "LocalWhitelist")
334 lwhitelist = pyzor.config.load_local_whitelist(lwhitelist_fp)
335 style = config.get("client", "Style")
336 for digested in get_input_handler(style):
337 if digested not in lwhitelist:
338 logger.critical("Digest %s is not whitelisted.", digested)
339 continue
340 lwhitelist.remove(digested)
341 with open(lwhitelist_fp, "w") as lwhitelist_f:
342 lwhitelist_f.write("\n".join(lwhitelist))
343 return True
344
345
346 def predigest(client, servers, config):
347 """Output the normalised version of each message, which is used to
348 create the digest.
349
350 This method can be used to diagnose which parts of the message are
351 used to determine uniqueness."""
352 for unused in get_input_handler(
353 "msg", digester=pyzor.digest.PrintingDataDigester):
354 pass
355 return True
356
357
358 def genkey(client, servers, config, hash_func=hashlib.sha1):
359 """Generate a key to use to authenticate pyzor requests. This method
360 will prompt for a password (and confirmation).
361
362 A random salt is generated (which makes it extremely difficult to
363 reverse the generated key to get the original password) and combined
364 with the entered password to provide a key. This key (but not the salt)
365 should be provided to the pyzord administrator, along with a username.
366 """
367 # pylint: disable-msg=W0613
368 password = getpass.getpass(prompt="Enter passphrase: ")
369 if getpass.getpass(prompt="Enter passphrase again: ") != password:
370 log = logging.getLogger("pyzor")
371 log.error("Passwords do not match.")
372 return False
373 # pylint: disable-msg=W0612
374 salt = "".join([chr(random.randint(0, 255))
375 for unused in xrange(hash_func(b"").digest_size)])
376 if sys.version_info >= (3, 0):
377 salt = salt.encode("utf8")
378 salt_digest = hash_func(salt)
379 pass_digest = hash_func(salt_digest.digest())
380 pass_digest.update(password.encode("utf8"))
381 print("salt,key:")
382 print("%s,%s" % (salt_digest.hexdigest(), pass_digest.hexdigest()))
383 return True
384
385
386 DISPATCHES = {
387 "ping": ping,
388 "pong": pong,
389 "info": info,
390 "check": check,
391 "report": report,
392 "whitelist": whitelist,
393 "digest": digest,
394 "predigest": predigest,
395 "genkey": genkey,
396 "local_whitelist": local_whitelist,
397 "local_unwhitelist": local_unwhitelist,
398 }
399
400
401 INPUT_HANDLERS = {
402 "msg": _get_input_msg,
403 "mbox": _get_input_mbox,
404 "digests": _get_input_digests,
405 }
406
407 if __name__ == "__main__":
408 main()