topicmod.parallel

Base classes for parallel model fitting. See the specific functions and classes in tm_gensim, tm_lda and tm_sklearn for parallel processing with popular topic modeling packages.

Markus Konrad <markus.konrad@wzb.eu>

Module Contents

Classes

MultiprocModelsRunner(self,worker_class,data,varying_parameters=None,constant_parameters=None,n_max_processes=None)
MultiprocModelsWorkerABC(self,worker_id,tasks_queue,results_queue,data,group=None,target=None,name=None,args=tuple,kwargs=None)
MultiprocEvaluationRunner(self,worker_class,available_metrics,data,varying_parameters,constant_parameters=None,metric=None,metric_options=None,n_max_processes=None,return_models=False)
MultiprocEvaluationWorkerABC(self,worker_id,eval_metric,eval_metric_options,return_models,tasks_queue,results_queue,data,group=None,target=None,name=None,args=tuple,kwargs=None)

Functions

_merge_params(varying_parameters,constant_parameters)
class MultiprocModelsRunner(worker_class, data, varying_parameters=None, constant_parameters=None, n_max_processes=None)
__init__(worker_class, data, varying_parameters=None, constant_parameters=None, n_max_processes=None)
__del__()

destructor. shutdown all workers

shutdown_workers()
run()
_setup_workers(worker_class)
_new_worker(worker_class, i, task_queue, results_queue, data)
_prepare_data()
class MultiprocModelsWorkerABC(worker_id, tasks_queue, results_queue, data, group=None, target=None, name=None, args=tuple, kwargs=None)
__init__(worker_id, tasks_queue, results_queue, data, group=None, target=None, name=None, args=tuple, kwargs=None)
run()
fit_model(data, params)
send_results(doc, params, results)
class MultiprocEvaluationRunner(worker_class, available_metrics, data, varying_parameters, constant_parameters=None, metric=None, metric_options=None, n_max_processes=None, return_models=False)
__init__(worker_class, available_metrics, data, varying_parameters, constant_parameters=None, metric=None, metric_options=None, n_max_processes=None, return_models=False)
_new_worker(worker_class, i, task_queue, results_queue, data)
class MultiprocEvaluationWorkerABC(worker_id, eval_metric, eval_metric_options, return_models, tasks_queue, results_queue, data, group=None, target=None, name=None, args=tuple, kwargs=None)
__init__(worker_id, eval_metric, eval_metric_options, return_models, tasks_queue, results_queue, data, group=None, target=None, name=None, args=tuple, kwargs=None)
_merge_params(varying_parameters, constant_parameters)