From 75fe3272b306235c02d7a19f9b5ba70fbb7838e0 Mon Sep 17 00:00:00 2001
From: Darko Poljak <darko.poljak@gmail.com>
Date: Fri, 11 Aug 2017 15:04:26 +0200
Subject: [PATCH] Add file locking for -j parallel execution.
---
cdist/config.py | 4 +--
cdist/core/manifest.py | 4 +--
cdist/emulator.py | 34 ++++++++++++-------
cdist/flock.py | 58 +++++++++++++++++++++++++++++++++
cdist/test/manifest/__init__.py | 2 +-
5 files changed, 85 insertions(+), 17 deletions(-)
create mode 100644 cdist/flock.py
diff --git a/cdist/config.py b/cdist/config.py
index b7ca1f84..cdff47eb 100644
--- a/cdist/config.py
+++ b/cdist/config.py
@@ -39,11 +39,9 @@ import cdist.hostsource
import cdist.exec.local
import cdist.exec.remote
-from cdist import inventory
-
import cdist.util.ipaddr as ipaddr
-from cdist import core
+from cdist import core, inventory
from cdist.util.remoteutil import inspect_ssh_mux_opts
diff --git a/cdist/core/manifest.py b/cdist/core/manifest.py
index 6f941550..0a26601a 100644
--- a/cdist/core/manifest.py
+++ b/cdist/core/manifest.py
@@ -113,8 +113,8 @@ class Manifest(object):
'__target_host_tags': self.local.target_host_tags,
}
- if self.log.getEffectiveLevel() == logging.DEBUG:
- self.env.update({'__cdist_debug': "yes"})
+ self.env.update(
+ {'__cdist_loglevel': str(self.log.getEffectiveLevel())})
def _open_logger(self):
self.log = logging.getLogger(self.target_host[0])
diff --git a/cdist/emulator.py b/cdist/emulator.py
index 7c9dfcca..abc37282 100644
--- a/cdist/emulator.py
+++ b/cdist/emulator.py
@@ -28,6 +28,7 @@ import sys
import cdist
from cdist import core
+from cdist import flock
class MissingRequiredEnvironmentVariableError(cdist.Error):
@@ -94,20 +95,25 @@ class Emulator(object):
"""Emulate type commands (i.e. __file and co)"""
self.commandline()
- self.setup_object()
- self.save_stdin()
- self.record_requirements()
- self.record_auto_requirements()
- self.log.trace("Finished %s %s" % (
- self.cdist_object.path, self.parameters))
+ self.init_object()
+
+ # locking for parallel execution
+ with flock.Flock(self.flock_path) as lock:
+ self.setup_object()
+ self.save_stdin()
+ self.record_requirements()
+ self.record_auto_requirements()
+ self.log.trace("Finished %s %s" % (
+ self.cdist_object.path, self.parameters))
def __init_log(self):
"""Setup logging facility"""
- if '__cdist_debug' in self.env:
- logging.root.setLevel(logging.DEBUG)
+ if '__cdist_loglevel' in self.env:
+ level = int(self.env['__cdist_loglevel'])
else:
- logging.root.setLevel(logging.INFO)
+ level = logging.OFF
+ logging.root.setLevel(level)
self.log = logging.getLogger(self.target_host[0])
@@ -150,8 +156,8 @@ class Emulator(object):
self.args = parser.parse_args(self.argv[1:])
self.log.trace('Args: %s' % self.args)
- def setup_object(self):
- # Setup object - and ensure it is not in args
+ def init_object(self):
+ # Initialize object - and ensure it is not in args
if self.cdist_type.is_singleton:
self.object_id = ''
else:
@@ -162,7 +168,13 @@ class Emulator(object):
self.cdist_object = core.CdistObject(
self.cdist_type, self.object_base_path, self.object_marker,
self.object_id)
+ lockfname = ('.' + self.cdist_type.name +
+ self.object_id + '_' +
+ self.object_marker + '.lock')
+ lockfname = lockfname.replace(os.sep, '_')
+ self.flock_path = os.path.join(self.object_base_path, lockfname)
+ def setup_object(self):
# Create object with given parameters
self.parameters = {}
for key, value in vars(self.args).items():
diff --git a/cdist/flock.py b/cdist/flock.py
new file mode 100644
index 00000000..d8bac916
--- /dev/null
+++ b/cdist/flock.py
@@ -0,0 +1,58 @@
+# -*- coding: utf-8 -*-
+#
+# 2017 Darko Poljak (darko.poljak at gmail.com)
+#
+# This file is part of cdist.
+#
+# cdist is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# cdist is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with cdist. If not, see <http://www.gnu.org/licenses/>.
+#
+#
+
+import fcntl
+import logging
+import os
+
+
+log = logging.getLogger('cdist-flock')
+
+
+class Flock():
+ def __init__(self, path):
+ self.path = path
+ self.lockfd = None
+
+ def flock(self):
+ log.debug('Acquiring lock on %s', self.path)
+ self.lockfd = open(self.path, 'w+')
+ fcntl.flock(self.lockfd, fcntl.LOCK_EX)
+ log.debug('Acquired lock on %s', self.path)
+
+ def funlock(self):
+ log.debug('Releasing lock on %s', self.path)
+ fcntl.flock(self.lockfd, fcntl.LOCK_UN)
+ self.lockfd.close()
+ self.lockfd = None
+ try:
+ os.remove(self.path)
+ except FileNotFoundError:
+ pass
+ log.debug('Released lock on %s', self.path)
+
+ def __enter__(self):
+ self.flock()
+ return self
+
+ def __exit__(self, *args):
+ self.funlock()
+ return False
diff --git a/cdist/test/manifest/__init__.py b/cdist/test/manifest/__init__.py
index e0da2d9f..95bf2768 100644
--- a/cdist/test/manifest/__init__.py
+++ b/cdist/test/manifest/__init__.py
@@ -136,7 +136,7 @@ class ManifestTestCase(test.CdistTestCase):
current_level = self.log.getEffectiveLevel()
self.log.setLevel(logging.DEBUG)
manifest = cdist.core.manifest.Manifest(self.target_host, self.local)
- self.assertTrue("__cdist_debug" in manifest.env)
+ self.assertTrue("__cdist_loglevel" in manifest.env)
self.log.setLevel(current_level)