"Fossies" - the Fresh Open Source Software Archive 
Member "pyzor-1.0.0/scripts/pyzord" (10 Dec 2014, 16510 Bytes) of package /linux/privat/pyzor-1.0.0.tar.gz:
As a special service "Fossies" has tried to format the requested source page into HTML format using (guessed) Python source code syntax highlighting (style:
standard) with prefixed line numbers.
Alternatively you can here
view or
download the uninterpreted source code file.
See also the latest
Fossies "Diffs" side-by-side code changes report for "pyzord":
0.9.0_vs_1.0.0.
1 #! /usr/bin/env python
2
3 """A front-end interface to the pyzor daemon."""
4
5 from __future__ import print_function
6
7 import os
8 import sys
9 import optparse
10 import traceback
11 import ConfigParser
12
13 import pyzor.client
14 import pyzor.config
15 import pyzor.server
16 import pyzor.engines
17 import pyzor.forwarder
18
19
20 def detach(stdout="/dev/null", stderr=None, stdin="/dev/null", pidfile=None):
21 """This forks the current process into a daemon.
22
23 The stdin, stdout, and stderr arguments are file names that
24 will be opened and be used to replace the standard file descriptors
25 in sys.stdin, sys.stdout, and sys.stderr.
26 These arguments are optional and default to /dev/null.
27 Note that stderr is opened unbuffered, so if it shares a file with
28 stdout then interleaved output may not appear in the order that you
29 expect."""
30 # Do first fork.
31 try:
32 pid = os.fork()
33 if pid > 0:
34 # Exit first parent.
35 sys.exit(0)
36 except OSError as err:
37 print("Fork #1 failed: (%d) %s" % (err.errno, err.strerror),
38 file=sys.stderr)
39 sys.exit(1)
40
41 # Decouple from parent environment.
42 os.chdir("/")
43 os.umask(0)
44 os.setsid()
45
46 # Do second fork.
47 try:
48 pid = os.fork()
49 if pid > 0:
50 # Exit second parent.
51 sys.exit(0)
52 except OSError as err:
53 print("Fork #2 failed: (%d) %s" % (err.errno, err.strerror),
54 file=sys.stderr)
55 sys.exit(1)
56
57 # Open file descriptors and print start message.
58 if not stderr:
59 stderr = stdout
60 stdi = open(stdin, "r")
61 stdo = open(stdout, "a+")
62 stde = open(stderr, "ab+", 0)
63 pid = str(os.getpid())
64 if pidfile:
65 with open(pidfile, "w+") as pidf:
66 pidf.write("%s\n" % pid)
67
68 # Redirect standard file descriptors.
69 os.dup2(stdi.fileno(), sys.stdin.fileno())
70 os.dup2(stdo.fileno(), sys.stdout.fileno())
71 os.dup2(stde.fileno(), sys.stderr.fileno())
72
73
74 def initialize_forwarding(client_config_dir, debug):
75 """Reads configuration and returns a pyzor client and the list of servers
76 where the digests should be forwarded to.
77
78 Returns the forwarder server.
79 """
80 forward_defaults = {
81 "ServersFile": "servers",
82 "AccountsFile": "accounts",
83 "LogFile": "",
84 "Timeout": "5", # seconds
85 "Style": "msg",
86 "ReportThreshold": "0",
87 "WhitelistThreshold": "0"
88 }
89 config = ConfigParser.ConfigParser()
90 config.add_section("client")
91 for key, value in forward_defaults.iteritems():
92 config.set("client", key, value)
93
94 config.read(os.path.join(client_config_dir, "config"))
95 homefiles = ["LogFile", "ServersFile", "AccountsFile"]
96
97 pyzor.config.expand_homefiles(homefiles, "client", client_config_dir,
98 config)
99 servers_fn = config.get("client", "ServersFile")
100 accounts_fn = config.get("client", "AccountsFile")
101 logger_fn = config.get("client", "LogFile")
102 timeout = int(config.get("client", "Timeout"))
103
104 # client logging must be set up before we call load_accounts
105 pyzor.config.setup_logging("pyzor", logger_fn, debug)
106
107 servers = pyzor.config.load_servers(servers_fn)
108 accounts = pyzor.config.load_accounts(accounts_fn)
109 client = pyzor.client.BatchClient(accounts, timeout)
110
111 return pyzor.forwarder.Forwarder(client, servers)
112
113
114 def load_configuration():
115 """Load the configuration for the server.
116
117 The configuration comes from three sources: the default values, the
118 configuration file, and command-line options."""
119 # Work out the default directory for configuration files.
120 # If $HOME is defined, then use $HOME/.pyzor, otherwise use /etc/pyzor.
121 userhome = os.getenv("HOME")
122 if userhome:
123 homedir = os.path.join(userhome, '.pyzor')
124 else:
125 homedir = os.path.join("/etc", "pyzor")
126
127 # Configuration defaults. The configuration file overrides these, and
128 # then the command-line options override those.
129 defaults = {
130 "Port": "24441",
131 "ListenAddress": "0.0.0.0",
132
133 "Engine": "gdbm",
134 "DigestDB": "pyzord.db",
135 "CleanupAge": str(60 * 60 * 24 * 30 * 4), # approximately 4 months
136
137 "Threads": "False",
138 "MaxThreads": "0",
139 "Processes": "False",
140 "MaxProcesses": "40",
141 "DBConnections": "0",
142 "PreFork": "0",
143 "Gevent": "False",
144
145 "ForwardClientHomeDir": "",
146
147 "PasswdFile": "pyzord.passwd",
148 "AccessFile": "pyzord.access",
149 "LogFile": "",
150 "SentryDSN": "",
151 "SentryLogLevel": "WARN",
152 "UsageLogFile": "",
153 "UsageSentryDSN": "",
154 "UsageSentryLogLevel": "WARN",
155 "PidFile": "pyzord.pid"
156 }
157
158 # Process any command line options.
159 description = "Listen for and process incoming Pyzor connections."
160 opt = optparse.OptionParser(description=description)
161 opt.add_option("-n", "--nice", dest="nice", type="int",
162 help="'nice' level", default=0)
163 opt.add_option("-d", "--debug", action="store_true", default=False,
164 dest="debug", help="enable debugging output")
165 opt.add_option("--homedir", action="store", default=homedir,
166 dest="homedir", help="configuration directory")
167 opt.add_option("-a", "--address", action="store", default=None,
168 dest="ListenAddress", help="listen on this IP")
169 opt.add_option("-p", "--port", action="store", type="int", default=None,
170 dest="Port", help="listen on this port")
171 opt.add_option("-e", "--database-engine", action="store", default=None,
172 dest="Engine", help="select database backend")
173 opt.add_option("--dsn", action="store", default=None, dest="DigestDB",
174 help="data source name (filename for gdbm, host,user,"
175 "password,database,table for MySQL)")
176 opt.add_option("--gevent", action="store", default=None, dest="Gevent",
177 help="set to true to use the gevent library")
178 opt.add_option("--threads", action="store", default=None, dest="Threads",
179 help="set to true if multi-threading should be used"
180 " (this may not apply to all engines)")
181 opt.add_option("--max-threads", action="store", default=None, type="int",
182 dest="MaxThreads", help="the maximum number of concurrent "
183 "threads (defaults to 0 which is "
184 "unlimited)")
185 opt.add_option("--processes", action="store", default=None,
186 dest="Processes", help="set to true if multi-processing "
187 "should be used (this may not apply "
188 "to all engines)")
189 opt.add_option("--max-processes", action="store", default=None, type="int",
190 dest="MaxProcesses", help="the maximum number of concurrent "
191 "processes (defaults to 40)")
192 opt.add_option("--db-connections", action="store", default=None, type="int",
193 dest="DBConnections", help="the number of db connections "
194 "that will be kept by the server."
195 " This only applies if threads "
196 "are used. Defaults to 0 which "
197 "means a new connection is used "
198 "for every thread. (this may not "
199 "apply all engines)")
200 opt.add_option("--pre-fork", action="store", default=None,
201 dest="PreFork", help="")
202 opt.add_option("--password-file", action="store", default=None,
203 dest="PasswdFile", help="name of password file")
204 opt.add_option("--access-file", action="store", default=None,
205 dest="AccessFile", help="name of ACL file")
206 opt.add_option("--cleanup-age", action="store", default=None,
207 dest="CleanupAge",
208 help="time before digests expire (in seconds)")
209 opt.add_option("--log-file", action="store", default=None,
210 dest="LogFile", help="name of the log file")
211 opt.add_option("--usage-log-file", action="store", default=None,
212 dest="UsageLogFile", help="name of the usage log file")
213 opt.add_option("--pid-file", action="store", default=None,
214 dest="PidFile", help="save the pid in this file after the "
215 "server is daemonized")
216 opt.add_option("--forward-client-homedir", action="store", default=None,
217 dest="ForwardClientHomeDir",
218 help="Specify a pyzor client configuration directory to "
219 "forward received digests to a remote pyzor server")
220 opt.add_option("--detach", action="store", default=None,
221 dest="detach", help="daemonizes the server and redirects "
222 "any output to the specified file")
223 opt.add_option("-V", "--version", action="store_true", default=False,
224 dest="version", help="print version and exit")
225 options, args = opt.parse_args()
226
227 if options.version:
228 print("%s %s" % (sys.argv[0], pyzor.__version__), file=sys.stderr)
229 sys.exit(0)
230
231 if len(args):
232 opt.print_help()
233 sys.exit()
234 try:
235 os.nice(options.nice)
236 except AttributeError:
237 pass
238
239 # Create the configuration directory if it doesn't already exist.
240 if not os.path.exists(options.homedir):
241 os.mkdir(options.homedir)
242
243 # Load the configuration.
244 config = ConfigParser.ConfigParser()
245 # Set the defaults.
246 config.add_section("server")
247 for key, value in defaults.iteritems():
248 config.set("server", key, value)
249 # Override with the configuration.
250 config.read(os.path.join(options.homedir, "config"))
251 # Override with the command-line options.
252 for key in defaults:
253 value = getattr(options, key, None)
254 if value is not None:
255 config.set("server", key, str(value))
256 return config, options
257
258
259 def main():
260 """Run the pyzor daemon."""
261 # Set umask - this restricts this process from granting any world access
262 # to files/directories created by this process.
263 os.umask(0077)
264
265 config, options = load_configuration()
266
267 homefiles = ["LogFile", "UsageLogFile", "PasswdFile", "AccessFile",
268 "PidFile"]
269
270 engine = config.get("server", "Engine")
271 database_classes = pyzor.engines.database_classes[engine]
272 use_gevent = config.get("server", "Gevent").lower() == "true"
273 use_threads = config.get("server", "Threads").lower() == "true"
274 use_processes = config.get("server", "Processes").lower() == "true"
275 use_prefork = int(config.get("server", "PreFork"))
276
277 if use_threads and use_processes:
278 print("You cannot use both processes and threads at the same time")
279 sys.exit(1)
280
281 # We prefer to use the threaded server, but some database engines
282 # cannot handle it.
283 if use_threads and database_classes.multi_threaded:
284 use_processes = False
285 database_class = database_classes.multi_threaded
286 elif use_processes and database_classes.multi_processing:
287 use_threads = False
288 database_class = database_classes.multi_processing
289 else:
290 use_threads = False
291 use_processes = False
292 database_class = database_classes.single_threaded
293
294 # If the DSN is a filename, then we make it absolute.
295 if database_class.absolute_source:
296 homefiles.append("DigestDB")
297
298 pyzor.config.expand_homefiles(homefiles, "server", options.homedir, config)
299
300 logger = pyzor.config.setup_logging("pyzord",
301 config.get("server", "LogFile"),
302 options.debug,
303 config.get("server", "SentryDSN"),
304 config.get("server", "SentryLogLevel"))
305 pyzor.config.setup_logging("pyzord-usage",
306 config.get("server", "UsageLogFile"),
307 options.debug,
308 config.get("server", "UsageSentryDSN"),
309 config.get("server", "UsageSentryLogLevel"))
310
311 db_file = config.get("server", "DigestDB")
312 passwd_fn = config.get("server", "PasswdFile")
313 access_fn = config.get("server", "AccessFile")
314 pidfile_fn = config.get("server", "PidFile")
315 address = (config.get("server", "ListenAddress"),
316 int(config.get("server", "port")))
317 cleanup_age = int(config.get("server", "CleanupAge"))
318
319 forward_client_home = config.get('server', 'ForwardClientHomeDir')
320 if forward_client_home:
321 forwarder = initialize_forwarding(forward_client_home, options.debug)
322 else:
323 forwarder = None
324
325 if use_gevent:
326 # Monkey patch the std libraries with gevent ones
327 try:
328 import signal
329 import gevent
330 import gevent.monkey
331 except ImportError as e:
332 logger.critical("Gevent library not found: %s", e)
333 sys.exit(1)
334 gevent.monkey.patch_all()
335 # The signal method does not get patched in patch_all
336 signal.signal = gevent.signal
337 # XXX The gevent libary might already be doing this.
338 # Enssure that all modules are reloaded so they benefit from
339 # the gevent library.
340 for module in (os, sys, pyzor, pyzor.server, pyzor.engines):
341 reload(module)
342
343 if options.detach:
344 detach(stdout=options.detach, pidfile=pidfile_fn)
345
346 if use_prefork:
347 if use_prefork < 2:
348 logger.critical("Pre-fork value cannot be lower than 2.")
349 sys.exit(1)
350 databases = database_class.get_prefork_connections(db_file, "c",
351 cleanup_age)
352 server = pyzor.server.PreForkServer(address, databases, passwd_fn,
353 access_fn, use_prefork)
354 elif use_threads:
355 max_threads = int(config.get("server", "MaxThreads"))
356 bound = int(config.get("server", "DBConnections"))
357
358 database = database_class(db_file, "c", cleanup_age, bound)
359 if max_threads == 0:
360 logger.info("Starting multi-threaded pyzord server.")
361 server = pyzor.server.ThreadingServer(address, database, passwd_fn,
362 access_fn, forwarder)
363 else:
364 logger.info("Starting bounded (%s) multi-threaded pyzord server.",
365 max_threads)
366 server = pyzor.server.BoundedThreadingServer(address, database,
367 passwd_fn, access_fn,
368 max_threads,
369 forwarder)
370 elif use_processes:
371 max_children = int(config.get("server", "MaxProcesses"))
372 database = database_class(db_file, "c", cleanup_age)
373 logger.info("Starting bounded (%s) multi-processing pyzord server.",
374 max_children)
375 server = pyzor.server.ProcessServer(address, database, passwd_fn,
376 access_fn, max_children, forwarder)
377 else:
378 database = database_class(db_file, "c", cleanup_age)
379 logger.info("Starting pyzord server.")
380 server = pyzor.server.Server(address, database, passwd_fn, access_fn,
381 forwarder)
382
383 if forwarder:
384 forwarder.start_forwarding()
385
386 try:
387 server.serve_forever()
388 except:
389 logger.critical("Failure: %s", traceback.format_exc())
390 finally:
391 logger.info("Server shutdown.")
392 server.server_close()
393 if forwarder:
394 forwarder.stop_forwarding()
395 if options.detach and os.path.exists(pidfile_fn):
396 try:
397 os.remove(pidfile_fn)
398 except Exception as e:
399 logger.warning("Unable to remove pidfile %r: %s",
400 pidfile_fn, e)
401
402
403 if __name__ == "__main__":
404 main()